#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # $Id$ # $HeadURL$ # $Author$ # # Ported to Python 3 for Qt6 Migration from __future__ import print_function from collections import defaultdict import datetime import getopt import re import subprocess import sys import tempfile import os import configparser class main_class(object): def __init__(self): self.project = None self.database = None self.db_host = "localhost" self.db_user = "postgres" self.database_backup = None self.from_backup = None # '/opt/postgres/db-setup/backup/guez_db_lts_ets_pro.sql' self.db_description = "Training Center EXCON " self.log_dir = None self.log_file_name = None if hasattr(sys, 'getwindowsversion'): # find postgresql version root = "C:/Program Files/PostgreSQL/" dirlist = [item for item in os.listdir(root) if os.path.isdir(os.path.join(root, item))] highestVersion = "0.0" for dirs in dirlist: if float(dirs) > float(highestVersion): highestVersion = str(dirs) psql_dir = os.path.join("C:\\Program Files\\PostgreSQL", highestVersion, "bin") self.psql_command = os.path.join(psql_dir, "psql.exe") self.createdb_command = os.path.join(psql_dir, "createdb.exe") self.dropdb_command = os.path.join(psql_dir, "dropdb.exe") print("\n Found psql in: ", psql_dir) else: self.psql_command = 'psql' self.createdb_command = 'createdb' self.dropdb_command = 'dropdb' self.with_exercise_data = False # Path, where sql file(s) for configuring the db are self.sql_path = None self.setting_path = None self.global_schemas = [] self.service_operator_rights = [] def psql_error_log_info(self, message): self.log(" Message: %s" % message) self.log(" => Database accessed by other users:") self.log(" current dir: %s" % os.getcwd()) self.log_hline() self.log("Open connection for postgresql: ", True) self.log_hline() self.system('%s postgres -h %s -U postgres -f check_connections.sql' % (self.psql_command, self.db_host), True) self.log_hline() sys.exit(2) def initialize(self): if self.project is None: print("No project given", file=sys.stderr) sys.exit(1) if self.log_dir is None: self.log_dir = '.' self.log_file_name = "%s/init_db_%s.log" % (self.log_dir, self.project) if self.setting_path is None: if hasattr(sys, 'getwindowsversion'): self.setting_path = "lts-cc.conf" # TODO: check path on Win else: self.setting_path = "/etc/lts-cc.conf" dbName, self.db_host = self.db_config(self.setting_path) if dbName is None: dbName = self.project + "_pro" if self.db_host is None: self.db_host = "localhost" if self.database is None: self.database = dbName if self.database_backup is None: self.database_backup = self.database + "_old" if self.sql_path is None: self.sql_path = "../../share/Lts/sql/" if self.from_backup is not None: if os.path.isfile(self.from_backup): self.log("# Restoring from backup: %s" % self.from_backup, True) self.with_exercise_data = False else: self.log("# Invalid backup file: %s" % self.from_backup, True) self.from_backup = None self.log_hline() self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True) self.log("# DATE : %s" % datetime.datetime.today(), True) self.log("# PROJECT : %s" % self.project, True) self.log("# DATABASE : %s" % self.database, True) if self.from_backup is not None: self.log("# FROM BACKUP : %s" % self.from_backup, True) else: self.log("# WITH TEST DATA : %s" % self.with_exercise_data, True) self.log_hline() def execute(self): self.log("# - Drop backup database %s ..." % self.database_backup, True) ret = self.system('%s -U postgres -h %s --if-exists %s' % (self.dropdb_command, self.db_host, self.database_backup)) if not ret: self.psql_error_log_info("Cannot drop database %s" % self.database_backup) self.log("# => dropdb finished", True) self.log_hline() # Check if db exists # -t: tuples only exists_db = self.system_returning_bool('%s -h %s -t -U postgres -c "SELECT CASE WHEN EXISTS ( SELECT datname FROM pg_database WHERE datname = \'%s\') THEN CAST(1 AS BIT) ELSE CAST(0 AS BIT) END " ' % (self.psql_command, self.db_host, self.database)) if exists_db: self.log("# - Rename existing database %s ..." % self.database, True) ret = self.system('%s -h %s -U postgres -c "ALTER DATABASE %s RENAME TO %s" postgres' % (self.psql_command, self.db_host, self.database, self.database_backup)) if not ret: self.psql_error_log_info("Failed to rename database %s" % self.database) self.log("# => Rename DB from %s to %s finished" % (self.database, self.database_backup), True) self.log_hline() self.log("# - Create database %s ..." % self.database, True) self.db_description = "Training Center EXCON (" + self.project + ")" ret = self.system('%s --host=%s -U postgres %s "%s"' % (self.createdb_command, self.db_host, self.database, self.db_description)) if not ret: self.psql_error_log_info("Failed to create database %s" % self.database) self.log("# => Database %s created" % self.database, True) self.log_hline() # GRANT create on database self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() self.log("# - Grant create on database %s to user ets_exm ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() # create users and grants (if they dont exists, function in postgres database) self.log("# - Creating default users on database %s ..." % self.database, True) self.execute_sql_file('init_db_users_grants.sql', self.db_user) # self.execute_sql_file('init_db_users_grants.sql', self.db_user) self.log("# => create default users finished", True) self.log_hline() if self.from_backup is not None: # restore from backup self.log("# - Restoring from backup %s ..." % self.from_backup, True) ret = self.system('%s -h %s -U postgres -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.from_backup, self.database, self.log_file_name)) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (self.from_backup, os.path.abspath(self.log_file_name))) self.log("# => Database %s restored from backup %s" % (self.database, self.from_backup), True) self.log_hline() self.log("# - Grant admin on database %s to user ets_exm ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT ets_admin TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.log("# - Grant all privileges on all tables and sequences to %s ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database)) self.log("# => grant finished", True) self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() self.log("# - Grant create on database %s to user ets_exm ..." % self.database, True) self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database)) self.log("# => grant finished", True) self.log_hline() # update procedures self.log("# - Update procedures on database %s ..." % self.database, True) self.execute_sql_file('init_db_procedures.sql', self.db_user) self.log("# => update procedures finished", True) self.log_hline() else: # create new db self.log("# START initialisation of database %s ..." % self.database, True) self.log_hline() # create procedures and schemas self.log("# - Creating default procedures and schemas on database %s ..." % self.database, True) sql_init_files = ['init_db_procedures.sql', 'init_db_schemas.sql'] for f in sql_init_files: self.execute_sql_file(f, self.db_user) self.log("# => create default procedures and schemas finished", True) self.log_hline() # Run sql script in specific directory if self.with_exercise_data: # make test-db self.create_test_exercise_schema() else: # make db files = self.get_files(self.sql_path) self.create_global_schemas() if self.project == 'guez': self.insert_test_data('whg') self.insert_test_data('ppm', 'types') self.insert_test_data('aes', 'types') self.insert_test_data('ppm', 'templates') self.insert_test_data('pdm') self.insert_test_data('pdm', 'translation') for f in self.service_operator_rights: self.execute_sql_file(f, "ets_opm") self.insert_test_data('opm', 'initial') self.insert_test_data('opm') self.insert_test_data('flm') self.insert_test_data('flm', 'file_content_san') self.insert_test_data('asm') self.insert_test_data('asm', 'workstations') self.insert_test_data('ism') self.insert_test_data('ism', 'asset') self.insert_test_data('avm') self.insert_test_data('avm', 'frequencysets') self.insert_test_data('som') # symbols in FLM fn = os.path.join('%s_flm_symbols.sql' % self.project) if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols inserted.", True) self.log_hline() fn = os.path.join('%s_update_flm_used_by.sql' % self.project) if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() else: fn = "update_flm_used_by.sql" if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() # end symbols self.log("# END initialisation for database %s." % self.database, True) self.log_hline() return 0 def log_hline(self): self.log("# -------------------------------------------------------------------------------", True) def log(self, logline, log_to_stdout=False): if self.log_file_name is None: print("Logfile name not yet set - log: %s" % logline, file=sys.stderr) return with open(self.log_file_name, 'a') as f: f.write(logline) f.write("\n") if log_to_stdout: print(logline) def die(self, message): self.log(message) print("fatal error: %s" % message, file=sys.stderr) sys.exit(3) def system(self, cmdstring, show_output=False): self.log("# Executing '%s'" % cmdstring, False) if hasattr(sys, 'getwindowsversion'): p = subprocess.Popen(cmdstring) ret = p.wait() else: p = subprocess.Popen(cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ret = p.wait() if show_output: error = p.stdout.read() print(error, file=sys.stderr) if ret != 0: print("Error executing %s" % cmdstring, file=sys.stderr) return ret == 0 def system_returning_bool(self, cmdstring): self.log("# Executing '%s'" % cmdstring) cmdstring = cmdstring.replace('-e', '-q') if hasattr(sys, 'getwindowsversion'): p = subprocess.Popen(cmdstring, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: p = subprocess.Popen(cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) ret = p.wait() error = p.stdout.read() bRet = error.strip() self.log(" bRet: '%s'" % bRet) if ret != 0: print("Error executing %s (%s)" % (cmdstring, error), file=sys.stderr) return bRet == b'1' def db_config(self, config_file): config = configparser.ConfigParser() self.log("# read from configFile: %s" % config_file) config.read(config_file) try: db_name = config.get('Global', 'DatabaseName') db_host = config.get('Global', 'DatabaseHostName') except Exception: print("Could not parse config file '%s'!" % config_file, file=sys.stderr) # sys.exit(1) db_name = None db_host = None return (db_name, db_host) def get_files(self, directory): ret = [] for path, subdirs, files in os.walk(directory): for name in files: if name.endswith('.sql'): complete_file_name = os.path.join(path, name) ret.append(complete_file_name) if '_' not in name: self.global_schemas.append(complete_file_name) self.log("# Found global schema: %s" % complete_file_name, False) if 'operator_rights' in name: self.service_operator_rights.append(complete_file_name) return ret def sort_global_schema_by_dependencies(self): self.log("# * Sorting global schemas by dependencies ...", True) # searching for dependencies pattern = re.compile(".+REFERENCES \\w\\w\\w\\.") # pattern thats used to determine the dependencies dependencies = defaultdict(list) for sqlFile in self.global_schemas: for line in open(sqlFile): for match in re.finditer(pattern, line): line = match.group() dependencyToFile = os.path.join(self.sql_path, line[len(line) - 4:len(line) - 1] + ".sql") if dependencyToFile != sqlFile: dependencies[sqlFile].append(dependencyToFile) # creating new schema list newFileList = list() # add files without dependencies for sqlFile in self.global_schemas: if sqlFile not in dependencies.keys(): newFileList.append(sqlFile) # add files with dependencies while len(dependencies) != 0: for key in dependencies.keys(): # print key, dependencies[key] if set(newFileList) >= set(dependencies[key]): newFileList.append(key) dependencies.pop(key) self.global_schemas = newFileList def create_global_schemas(self): self.sort_global_schema_by_dependencies() self.log("# * Create global schemas ...", True) for f in self.global_schemas: schema_name = os.path.basename(f) user_name = "ets_" + os.path.splitext(schema_name)[0] if os.path.getsize(self.sql_path + '/' + schema_name) > 0: self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), False) self.execute_sql_file(self.sql_path + '/' + schema_name, user_name) self.log("# => Global schemas created.", True) self.log_hline() def create_global_schema(self, svc_name): self.log_hline() self.log("# * Create global schema: " + svc_name, True) for f in self.global_schemas: schema_name = os.path.basename(f) user_name = "ets_" + os.path.splitext(schema_name)[0] if os.path.splitext(schema_name)[0] == svc_name and os.path.getsize(self.sql_path + '/' + schema_name) > 0: self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), True) self.execute_sql_file(self.sql_path + '/' + schema_name, user_name) self.log("# Global schema created.", True) def insert_test_data(self, svc_name, fn_postfix=None, db_user=None): if fn_postfix: postfix = '_' + fn_postfix postfix_log = ' (' + fn_postfix + ')' else: postfix = '' postfix_log = '' fn = os.path.join('%s_%s_values%s.sql' % (self.project, svc_name, postfix)) fn_proj = fn if not os.path.isfile(fn): fn = os.path.join('insert_%s_values%s.sql' % (svc_name, postfix)) if db_user is None: db_user = 'ets_%s' % svc_name if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, db_user) self.log("# => %s%s Values inserted." % (svc_name.upper(), postfix_log), True) else: self.log("# FILE NOT FOUND: %s (or %s)" % (fn, fn_proj), True) self.log_hline() def sorted_services(self, sqlFiles): self.log("# * Sorting services ...", True) pattern = re.compile("^-- DEPENDS: (.*)$") dependencies = defaultdict(list) for sqlFile, svcName in sqlFiles: for line in open(sqlFile): for match in re.finditer(pattern, line): service = match.group(1) dependencies[svcName].append(service) # creating new schema list self.log("# * Creating new schema list ...", True) newSvcList = list() # add files without dependencies self.log("# * Add files without dependencies ...", True) for sqlFile, svcName in sqlFiles: if svcName not in dependencies.keys(): newSvcList.append(svcName) # add files with dependencies # print "services without deps: " # print newSvcList # print "services with deps:" # print dependencies self.log("# * Add files with dependencies ...", True) while len(dependencies) != 0: for key in dependencies.keys(): # >=: Test whether every element in other is in the set. if frozenset(newSvcList) >= frozenset(dependencies[key]): newSvcList.append(key) dependencies.pop(key) return newSvcList def create_test_exercise_schema(self): # Create schema for Test exercise (ex_42, ex_35) # Add Test User (ex_42, ex_35), grant privileges fn = "test_exm_exercise.sql" if os.path.isfile(fn): self.log("# * Create test exercises (35, 42) ...", True) self.execute_sql_file(fn, self.db_user) self.log("# => Test users and schemas created.", True) self.log_hline() files = self.get_files(self.sql_path) self.create_global_schemas() exercise_sql_files = list() for f in files: fn = os.path.basename(f) svc_name = fn.split('_')[0].split('/')[-1] if "42" in f: exercise_sql_files.append((f, svc_name)) # sort exercise dependent sql files ("-- DEPENDS: ") exercise_svc_sorted = self.sorted_services(exercise_sql_files) for ex_id in (35, 42): # username = 'ets_ex_%s' % ex_id username = 'ets_exm' for svc in exercise_svc_sorted: filename = os.path.join(self.sql_path, '%s_ex_%s.sql' % (svc, ex_id)) self.execute_sql_file(filename, username) # make test-db self.insert_test_data('pdm') self.insert_test_data('pdm', 'translation') self.insert_test_data('asm') # needs to be exeucted after pdm because of foreign key self.insert_test_data('asm', 'test') # military self.insert_test_data('exm') # needs to be executed after pdm, because of foreign key for f in self.service_operator_rights: self.execute_sql_file(f, "ets_opm") self.insert_test_data('opm', 'initial') self.insert_test_data('opm') self.insert_test_data('opm', 'test') self.insert_test_data('flm') self.insert_test_data('flm', 'test') self.insert_test_data('flm', 'file_content_san') self.insert_test_data('ism', 'test') self.insert_test_data('ism', 'asset') self.insert_test_data('avm') self.insert_test_data('avm', 'frequencysets') self.insert_test_data('avm', 'test') self.insert_test_data('som') if self.project == 'guez': self.insert_test_data('whg') self.insert_test_data('ppm', 'types') self.insert_test_data('ppm', 'templates') self.insert_test_data('ppm', 'ex_35', 'ets_exm') self.insert_test_data('ppm', 'ex_42', 'ets_exm') self.insert_test_data('aes', 'types') self.insert_test_data('aes', 'ex_35', 'ets_ex_00') self.insert_test_data('aes', 'ex_42', 'ets_ex_00') # self.insert_test_data('oss', 'ex_35', 'ets_ex_00') self.insert_test_data('oss', 'ex_42', 'ets_ex_00') self.insert_test_data('rep', 'ex_42', 'ets_ex_00') fn = os.path.join('%s_flm_symbols.sql' % self.project) if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols inserted.", True) self.log_hline() fn = os.path.join('%s_flm_update_used_by.sql' % self.project) if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() else: fn = "update_flm_used_by.sql" if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, self.db_user) self.log("# => Symbols updated.", True) self.log_hline() # civil # self.insert_test_data('pdm') # self.insert_test_data('ppm') # self.insert_test_data('ppm', 'ex_42', 'ets_ex_00') # needs to be executed as super user postgres fn = 'update_images_opm.sql' if os.path.isfile(fn): self.log("# Executing sql: %s" % fn, True) self.execute_sql_file(fn, 'postgres') fn = 'test_scenario_participant_positions.py' if os.path.isfile(fn): import test_scenario_participant_positions self.log("# Executing sql: %s" % fn, True) self.run_sql_statements(test_scenario_participant_positions.generate_participant_positions()) fn = 'test_scenario_participant_state.py' if os.path.isfile(fn): import test_scenario_participant_state self.log("# Executing sql: %s" % fn, True) self.run_sql_statements(test_scenario_participant_state.generate_participant_state()) def execute_sql_file(self, f, db_user): self.log("Run sql-file %s as db-user %s" % (f, db_user), False) ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, db_user, f, self.database, self.log_file_name)) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name))) def run_sql_statements(self, sql_stmts): f = tempfile.NamedTemporaryFile(delete=False) f.write('\n'.join(sql_stmts).encode('utf-8')) f.close() self.execute_sql_file(f.name, 'postgres') os.remove(f.name) def usage(): print("Usage: %s " % sys.argv[0]) print("usage: $0 -p -d -h -a -c -b -s -f ") print(" -p (--project) : guez / ahtc / qatar / l400p2 ") print(" -d (--database) : database name (default lts_ets_pro)") print(" -h (--host) : database server hostname (default localhost)") print(" -a (--archive-as-name) : Name of archive database") print(" Old database will be renamed to the archive-name (Default _old) ") print(" -e (--exercise-data) : with exercise (test) data ") print(" -c (--config-path) : config-path - sql-files for configuring the database ") print(" -b (--base-path) : base-path - config files (py-script & other sql-base scripts)") print(" -s (--setting-path) : setting-path - path where the default database-settings are stored (Default: /etc/lts-cc.conf)") print(" -f (--from-backup) : using a backup file") def main(): p = main_class() try: opts, args = getopt.getopt(sys.argv[1:], ":p:d:h:a:s:c:eb:f:", ["project", "database", "db-host", "archive-as-name", "setting-path", "config-path", "exercise-data", "base-path", "from-backup"]) except getopt.GetoptError as err: print(err, file=sys.stderr) usage() sys.exit(2) for o, a in opts: if o in ("-p", "--project"): p.project = a if o in ("-d", "--database"): p.database = a if o in ("-h", "--db-host"): p.db_host = a if o in ("-a", "--archive-as-name"): p.save_as = a if o in ("-s", "--setting-path"): p.setting_path = a if o in ("-c", "--config-path"): p.sql_path = a if o in ("-e", "--exercise-data"): p.with_exercise_data = True if o in ("-b", "--base-path"): p.base_path = a if o in ("-f", "--from-backup"): p.from_backup = a p.initialize() p.execute() sys.exit(0) if __name__ == "__main__": main()