#!/usr/bin/python # -*- coding: utf-8 -*- # # $Id$ # $HeadURL$ # $Author$ # from __future__ import print_function from collections import defaultdict import datetime import getopt import re import subprocess import sys import tempfile import os import string import ConfigParser class main_class(object): def __init__( self ): self.project = None self.database = None self.db_host = "localhost" self.db_user = "postgres" self.database_backup = None self.from_backup = None #'/opt/postgres/db-setup/backup/guez_db_lts_ets_pro.sql' self.db_description = "Training Center EXCON " self.log_dir = None self.log_file_name = None if hasattr(sys, 'getwindowsversion'): #find postgresql version root="C:/Program Files/PostgreSQL/" dirlist = [ item for item in os.listdir(root) if os.path.isdir(os.path.join(root, item)) ] highestVersion = "0.0" for dirs in dirlist: if float(dirs) > float(highestVersion): highestVersion = str(dirs) psql_dir = os.path.join("C:\Program Files\PostgreSQL" , highestVersion, "bin") self.psql_command = os.path.join(psql_dir, "psql.exe") self.createdb_command = os.path.join(psql_dir, "createdb.exe") self.dropdb_command = os.path.join(psql_dir, "dropdb.exe") print("\n Found psql in: ", psql_dir ) else: self.psql_command = 'psql' self.createdb_command = 'createdb' self.dropdb_command = 'dropdb' self.with_exercise_data = False # Path, where sql file(s) for configuring the db are self.sql_path = None self.setting_path = None self.global_schemas = [] self.service_operator_rights = [] def psql_error_log_info(self, message): self.log(" Message: %s" % message ) self.log(" => Database accessed by other users:") self.log(" current dir: %s" % os.getcwd() ) self.log_hline() self.log("Open connection for postgresql: ", True) self.log_hline() self.system('%s postgres -h %s -U postgres -f check_connections.sql' % ( self.psql_command, self.db_host ), True) self.log_hline() sys.exit( 2 ) def initialize(self): if self.project == None: print("No project given", file=sys.stderr) sys.exit(1) if self.log_dir == None: self.log_dir ='.' self.log_file_name = "%s/init_db_%s.log" % (self.log_dir, self.project) if self.setting_path == None: if hasattr(sys, 'getwindowsversion'): self.setting_path = "lts-cc.conf" #TODO: check path on Win else: self.setting_path = "/etc/lts-cc.conf" dbName, self.db_host = self.db_config(self.setting_path) if dbName == None: dbName = self.project + "_pro" if self.db_host == None: self.db_host = "localhost" if self.database == None: self.database = dbName if self.database_backup == None: self.database_backup = self.database + "_old" if self.sql_path == None: self.sql_path = "../../share/Lts/sql/" if self.from_backup != None: if (os.path.isfile(self.from_backup)): self.log("# Restoring from backup: %s" % self.from_backup, True) self.with_exercise_data = False else: self.log("# Invalid backup file: %s" % self.from_backup, True) self.from_backup = None self.log_hline() self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True) self.log("# DATE : %s" % datetime.datetime.today(), True) self.log("# PROJECT : %s" % self.project, True) self.log("# DATABASE : %s" % self.database, True) if self.from_backup != None: self.log("# FROM BACKUP : %s" % self.from_backup, True) else: self.log("# WITH TEST DATA : %s" % self.with_exercise_data, True) self.log_hline() def execute(self): self.log("# - Drop backup database %s ..." % self.database_backup, True ) ret = self.system('%s -U postgres -h %s --if-exists %s' % (self.dropdb_command, self.db_host, self.database_backup)) if not ret: self.psql_error_log_info("Cannot drop database %s" % self.database_backup ) self.log("# => dropdb finished", True) self.log_hline() # Check if db exists # -t: tuples only exists_db = self.system_returning_bool('%s -h %s -t -U postgres -c "SELECT CASE WHEN EXISTS ( SELECT datname FROM pg_database WHERE datname = \'%s\') THEN CAST(1 AS BIT) ELSE CAST(0 AS BIT) END " ' % (self.psql_command, self.db_host, self.database) ) if exists_db: self.log("# - Rename existing database %s ..." % self.database, True ) ret = self.system('%s -h %s -U postgres -c "ALTER DATABASE %s RENAME TO %s" postgres' % (self.psql_command, self.db_host, self.database, self.database_backup)) if not ret: self.psql_error_log_info("Failed to rename database %s" % self.database ) self.log("# => Rename DB from %s to %s finished" % (self.database, self.database_backup), True ) self.log_hline() self.log("# - Create database %s ..." % self.database, True ) self.db_description = "Training Center EXCON (" + self.project + ")" ret = self.system('%s --host=%s -U postgres %s "%s"' % (self.createdb_command, self.db_host, self.database, self.db_description)) if not ret: self.psql_error_log_info("Failed to create database %s" % self.database ) self.log("# => Database %s created" % (self.database), True) self.log_hline() # GRANT create on database self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() self.log("# - Grant create on database %s to user etx_exm ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() # create users and grants (if they dont exists, function in postgres database) self.log("# - Creating default users on database %s ..." % self.database, True) self.system('%s -h %s -U postgres -f "init_db_users_grants.sql" %s' % (self.psql_command, self.db_host, 'postgres')) # self.execute_sql_file('init_db_users_grants.sql', self.db_user) self.log("# => create default users finished", True) self.log_hline() if self.from_backup != None : # restore from backup self.log("# - Restoring from backup %s ..." % self.from_backup, True ) ret = self.system('%s -h %s -U postgres -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.from_backup, self.database, self.log_file_name)) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (self.from_backup, os.path.abspath(self.log_file_name))) self.log("# => Database %s restored from backup %s" % (self.database, self.from_backup), True) self.log_hline() self.log("# - Grant admin on database %s to user ets_exm ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT ets_admin TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.log("# - Grant all privileges on all tables and sequences to %s ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.log("# => grant finished", True) self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() self.log("# - Grant create on database %s to user ets_exm ..." % self.database, True ) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() # update procedures self.log("# - Update procedures on database %s ..." % self.database, True) self.execute_sql_file('init_db_procedures.sql', self.db_user) self.log("# => update procedures finished", True) self.log_hline() else: # create new db self.log("# START initialisation of database %s ..." % self.database, True) self.log_hline() # create procedures and schemas self.log("# - Creating default procedures and schemas on database %s ..." % self.database, True) sql_init_files = [ 'init_db_procedures.sql', 'init_db_schemas.sql'] for f in sql_init_files: self.execute_sql_file(f, self.db_user) self.log("# => create default procedures and schemas finished", True) self.log_hline() # Run sql script in specific directory if self.with_exercise_data: # make test-db self.create_test_exercise_schema() else: # make db files = self.get_files(self.sql_path) self.create_global_schemas() if (self.project == 'guez'): self.insert_test_data('whg') self.insert_test_data('ppm', 'types') self.insert_test_data('aes', 'types') self.insert_test_data('ppm', 'templates') self.insert_test_data('pdm') self.insert_test_data('pdm', 'translation') for f in self.service_operator_rights: self.execute_sql_file(f, "ets_opm") self.insert_test_data('opm', 'initial') self.insert_test_data('opm') self.insert_test_data('flm') self.insert_test_data('flm', 'file_content_san') self.insert_test_data('asm') self.insert_test_data('asm', 'workstations') self.insert_test_data('ism') self.insert_test_data('ism', 'asset') self.insert_test_data('avm') self.insert_test_data('avm', 'frequencysets') self.insert_test_data('som') # symbols in FLM fn = os.path.join('%s_flm_symbols.sql' % self.project) if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols inserted.", True) self.log_hline() fn = os.path.join('%s_update_flm_used_by.sql' % self.project) if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() else: fn = "update_flm_used_by.sql" if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file( fn , self.db_user) self.log("# => Symbols updated.", True) self.log_hline() # end symbols self.log("# END initialisation for database %s." % self.database, True) self.log_hline() return 0 def log_hline(self): self.log("# -------------------------------------------------------------------------------", True) def log(self, logline, log_to_stdout = False): if self.log_file_name == None: print("Logfile name not yet set - log: %s" % logline, file=sys.stderr) return with open( self.log_file_name, 'a' ) as f: f.write(logline) f.write("\n") if log_to_stdout: print(logline) def die( self, message ): self.log( message ) print("fatal error: %s" % message, file=sys.stderr) sys.exit( 3 ) def system( self, cmdstring, show_output = False ): self.log( "# Executing '%s'" % cmdstring, False) if hasattr(sys, 'getwindowsversion'): p = subprocess.Popen( cmdstring ) ret = p.wait() else: p = subprocess.Popen( cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) ret = p.wait() if show_output: error = p.stdout.read() print(error, file=sys.stderr) if ( ret != 0 ): print("Error executing %s" % cmdstring, file=sys.stderr) return ret == 0 def system_returning_bool( self, cmdstring ): self.log( "# Executing '%s'" % cmdstring ) cmdstring = string.replace(cmdstring, '-e', '-q') if hasattr(sys, 'getwindowsversion'): p = subprocess.Popen( cmdstring, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) else: p = subprocess.Popen( cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) ret = p.wait() error = p.stdout.read() bRet = error.strip() self.log(" bRet: '%s'" % bRet ) if ( ret != 0 ): print("Error executing %s (%s)" % (cmdstring, error), file=sys.stderr) return bRet == '1' def db_config(self, config_file ): config = ConfigParser.ConfigParser() self.log("# read from configFile: %s" % config_file ) config.read(config_file) try: db_name = config.get('Global', 'DatabaseName') db_host = config.get('Global', 'DatabaseHostName') except: print("Could not parse config file '%s'!" % config_file, file=sys.stderr) #sys.exit(1) db_name = None db_host = None return (db_name, db_host) def get_files( self, directory ): ret = [] for path, subdirs, files in os.walk( directory ): for name in files: if name.endswith('.sql'): complete_file_name = os.path.join(path, name) ret.append( complete_file_name ) if '_' not in name: self.global_schemas.append(complete_file_name) self.log("# Found global schema: %s" % complete_file_name, False) if 'operator_rights' in name: self.service_operator_rights.append(complete_file_name) return ret def sort_global_schema_by_dependencies(self): self.log("# * Sorting global schemas by dependencies ...", True) # searching for dependencies pattern = re.compile(".+REFERENCES \w\w\w\.") #pattern thats used to determine the dependencies dependencies = defaultdict(list) for sqlFile in self.global_schemas: for line in open(sqlFile): for match in re.finditer(pattern, line): line = match.group() dependencyToFile = os.path.join(self.sql_path, line[len(line)-4:len(line)-1]+".sql") if dependencyToFile != sqlFile: dependencies[sqlFile].append(dependencyToFile) #creating new schema list newFileList = list() # add files without dependencies for sqlFile in self.global_schemas: if sqlFile not in dependencies.keys(): newFileList.append(sqlFile) #add files with dependencies while not 0 == len(dependencies): for key in dependencies.keys(): #print key, dependencies[key] if set(newFileList) >= set(dependencies[key]): newFileList.append(key) dependencies.pop(key) self.global_schemas = newFileList def create_global_schemas(self): self.sort_global_schema_by_dependencies() self.log("# * Create global schemas ...", True) for f in self.global_schemas: schema_name = os.path.basename(f) user_name = "ets_" + os.path.splitext(schema_name)[0] if (os.path.getsize(self.sql_path + '/' + schema_name) > 0): self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), False) self.execute_sql_file( self.sql_path + '/' + schema_name, user_name) self.log("# => Global schemas created.", True) self.log_hline() def create_global_schema(self, svc_name): self.log_hline() self.log("# * Create global schema: " + svc_name, True) for f in self.global_schemas: schema_name = os.path.basename(f) user_name = "ets_" + os.path.splitext(schema_name)[0] if (os.path.splitext(schema_name)[0] == svc_name and os.path.getsize(self.sql_path + '/' + schema_name) > 0): self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), True) self.execute_sql_file( self.sql_path + '/' + schema_name, user_name) self.log("# Global schema created.", True) def insert_test_data(self, svc_name, fn_postfix = None, db_user = None): if fn_postfix: postfix = '_' + fn_postfix postfix_log = ' (' + fn_postfix + ')' else: postfix = '' postfix_log = '' fn = os.path.join('%s_%s_values%s.sql' % (self.project, svc_name, postfix)) fn_proj = fn; if (not os.path.isfile(fn)): fn = os.path.join('insert_%s_values%s.sql' % (svc_name, postfix)) if not db_user: db_user = 'ets_%s' % svc_name if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file( fn, db_user) self.log("# => %s%s Values inserted." % ( svc_name.upper(), postfix_log ), True) else: self.log("# FILE NOT FOUND: %s (or %s)" % (fn, fn_proj) , True) self.log_hline() def sorted_services(self, sqlFiles): self.log("# * Sorting services ...", True) pattern = re.compile("^-- DEPENDS: (.*)$") dependencies = defaultdict(list) for sqlFile,svcName in sqlFiles: for line in open(sqlFile): for match in re.finditer(pattern, line): service = match.group(1) dependencies[svcName].append(service) #creating new schema list self.log("# * Creating new schema list ...", True) newSvcList = list() # add files without dependencies self.log("# * Add files without dependencies ...", True) for sqlFile,svcName in sqlFiles: if svcName not in dependencies.keys(): newSvcList.append(svcName) #add files with dependencies #print "services without deps: " #print newSvcList #print "services with deps:" #print dependencies self.log("# * Add files with dependencies ...", True) while not 0 == len(dependencies): for key in dependencies.keys(): # >=: Test whether every element in other is in the set. if frozenset(newSvcList) >= frozenset(dependencies[key]): newSvcList.append(key) dependencies.pop(key) return newSvcList def create_test_exercise_schema(self): # Create schema for Test exercise (ex_42, ex_35) # Add Test User (ex_42, ex_35), grant privileges fn = "test_exm_exercise.sql" if (os.path.isfile(fn)): self.log("# * Create test exercises (35, 42) ...", True) self.execute_sql_file(fn, self.db_user) self.log("# => Test users and schemas created.", True) self.log_hline() files = self.get_files(self.sql_path) self.create_global_schemas() exercise_sql_files = list() for f in files: fn = os.path.basename(f) svc_name = fn.split('_')[0].split('/')[-1] if "42" in f: exercise_sql_files.append((f, svc_name)) # sort exercise dependent sql files ("-- DEPENDS: ") exercise_svc_sorted = self.sorted_services(exercise_sql_files) for ex_id in (35,42): # username = 'ets_ex_%s' % ex_id username = 'ets_exm' for svc in exercise_svc_sorted: filename = os.path.join(self.sql_path, '%s_ex_%s.sql' % ( svc, ex_id )) self.execute_sql_file(filename, username) # make test-db self.insert_test_data('pdm') self.insert_test_data('pdm', 'translation') self.insert_test_data('asm') # needs to be exeucted after pdm because of foreign key self.insert_test_data('asm', 'test') # military self.insert_test_data('exm') # needs to be executed after pdm, because of foreign key for f in self.service_operator_rights: self.execute_sql_file(f, "ets_opm") self.insert_test_data('opm', 'initial') self.insert_test_data('opm') self.insert_test_data('opm', 'test') self.insert_test_data('flm') self.insert_test_data('flm', 'test') self.insert_test_data('flm', 'file_content_san') self.insert_test_data('ism', 'test') self.insert_test_data('ism', 'asset') self.insert_test_data('avm') self.insert_test_data('avm', 'frequencysets') self.insert_test_data('avm', 'test') self.insert_test_data('som') if (self.project == 'guez'): self.insert_test_data('whg') self.insert_test_data('ppm', 'types') self.insert_test_data('ppm', 'templates') self.insert_test_data('ppm', 'ex_35', 'ets_exm') self.insert_test_data('ppm', 'ex_42', 'ets_exm') self.insert_test_data('aes', 'types') self.insert_test_data('aes', 'ex_35', 'ets_ex_00') self.insert_test_data('aes', 'ex_42', 'ets_ex_00') #self.insert_test_data('oss', 'ex_35', 'ets_ex_00') self.insert_test_data('oss', 'ex_42', 'ets_ex_00') self.insert_test_data('rep', 'ex_42', 'ets_ex_00') fn = os.path.join('%s_flm_symbols.sql' % self.project) if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols inserted.", True) self.log_hline() fn = os.path.join('%s_flm_update_used_by.sql' % self.project) if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() else: fn = "update_flm_used_by.sql" if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file( fn , self.db_user) self.log("# => Symbols updated.", True) self.log_hline() # civil #self.insert_test_data('pdm') #self.insert_test_data('ppm') #self.insert_test_data('ppm', 'ex_42', 'ets_ex_00') # needs to be executed as super user postgres fn = 'update_images_opm.sql' if (os.path.isfile(fn)): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, 'postgres') fn = 'test_scenario_participant_positions.py' if (os.path.isfile(fn)): import test_scenario_participant_positions self.log("# Executing sql: %s" % fn, True) self.run_sql_statements(test_scenario_participant_positions.generate_participant_positions()) fn = 'test_scenario_participant_state.py' if (os.path.isfile(fn)): import test_scenario_participant_state self.log("# Executing sql: %s" % fn, True) self.run_sql_statements(test_scenario_participant_state.generate_participant_state()) def execute_sql_file(self, f, db_user): self.log( "Run sql-file %s as db-user %s" % (f, db_user), False) ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, db_user, f, self.database, self.log_file_name)) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name))) def run_sql_statements(self, sql_stmts): f = tempfile.NamedTemporaryFile(delete = False) f.write('\n'.join(sql_stmts)) f.close() self.execute_sql_file(f.name, 'postgres') os.remove(f.name) def usage(): print("Usage: %s " % sys.argv[0]) print("usage: $0 -p -d -h -a -c -b -s -f ") print(" -p (--project) : guez / ahtc / qatar / l400p2 ") print(" -d (--database) : database name (default lts_ets_pro)") print(" -h (--host) : database server hostname (default localhost)") print(" -a (--archive-as-name) : Name of archive database") print(" Old database will be renamed to the archive-name (Default _old) ") print(" -e (--exercise-data) : with exercise (test) data ") print(" -c (--config-path) : config-path - sql-files for configuring the database ") print(" -b (--base-path) : base-path - config files (py-script & other sql-base scripts)") print(" -s (--setting-path) : setting-path - path where the default database-settings are stored (Default: /etc/lts-cc.conf)") print(" -f (--from-backup) : using a backup file") def main(): p = main_class() try: opts, args = getopt.getopt(sys.argv[1:], ":p:d:h:a:s:c:e:b:f:", ["project", "database", "db-host", "archive-as-name", "setting-path", "config-path", "exercise-data", "base-path", "from-backup"]) except getopt.GetoptError, err: print(err, file=sys.stderr) usage() sys.exit(2) for o, a in opts: if o in ("-p", "--project"): p.project = a if o in ("-d", "--database"): p.database = a if o in ("-h", "--db-host"): p.db_host= a if o in ("-a", "--archive-as-name"): p.save_as = a if o in ("-s", "--setting-path"): p.setting_path = a if o in ("-c", "--config-path"): p.sql_path = a if o in ("-e", "--exercise-data"): p.with_exercise_data = 'True' if o in ("-b", "--base-path"): p.base_path = a if o in ("-f", "--from-backup"): p.from_backup = a p.initialize() p.execute() sys.exit(0) if __name__ == "__main__": main()