#!/usr/bin/python # -*- coding: utf-8 -*- # # $Id$ # $HeadURL$ # $Author$ # from __future__ import print_function from collections import defaultdict import datetime import subprocess import sys import os class main_class(object): def __init__( self ): self.project = "guez" self.database = "lts_ets_pro" self.db_host = "localhost" self.db_user = "postgres" self.log_dir = '.' self.log_file_name = "%s/update_db_%s.log" % (self.log_dir, self.project) self.psql_command = 'psql' self.pg_dump_command = 'pg_dump' self.csv_file = 'guez_db_update_charge_types.csv' self.chargeTypes = {} def initialize(self): self.log_hline() self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True) self.log("# DATE : %s" % datetime.datetime.today(), True) self.log("# PROJECT : %s" % self.project, True) self.log("# DATABASE : %s" % self.database, True) self.log_hline() def execute(self): # execute sql files self.log("# - Executings sql files on database %s ..." % self.database, True) self.log_hline() f = 'guez_db_update_charge_types_before.sql' self.log("# - Executing sql file %s ..." % f, True) self.execute_sql_file(f) self.log("# => %s sql finished" % f, True) self.log_hline() self.log("# - Load charge types csv file %s" % self.csv_file, True) self.chargeTypes = self.loadChargeTypes() self.log_hline() f = 'guez_db_create_charge_types_foreign_keys.sql' exercises = ['ex_14', 'ex_17', 'ex_21', 'ex_22', 'ex_26', 'ex_27', 'ex_28', 'ex_29', 'ex_30', 'ex_31', 'ex_32', 'ex_33', 'ex_34', 'ex_35', 'ex_36', 'ex_37', 'ex_38', 'ex_39', 'ex_40', 'ex_41', 'ex_42', 'ex_44', 'ex_45', 'ex_46', 'ex_51'] tables = ['eem_event', 'rec_dynamic_participant_data', 'rec_history_state', 'rec_dus_mine_state', 'rec_equipment_gear_charge_state'] for ex in exercises: self.dump_dynamic_tables(ex, tables) self.log("# - Creating foreign keys (sql file %s) in schema %s" % (f, ex), True) self.execute_sql_file_schema(f, ex) self.log("# => %s finished for exercise %s " % (f, ex), True) self.log_hline() f = 'guez_db_update_charge_types_after.sql' self.log("# - UPDATING ids - executing sql file %s ..." % f, True) self.execute_sql_file(f) self.log("# => %s sql finished" % f, True) self.log_hline() for ex in exercises: for tabl in tables: # replace charge types old_file = ex+"_"+tabl+".sql" new_file = ex+"_"+tabl+"_new.sql" self.log("# - Replace charge types in file %s" % old_file, True) db = self.read(old_file) db = self.replaceChargeTypes(db) self.write(new_file, db) # delete from tables self.log("# - Deleting dynamic data in schema %s" % ex, True) cmd = '%s -h %s -U %s -c "DELETE FROM %s;" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, tabl, self.database, ex, self.log_file_name) self.log(" Running command %s" % cmd, True) ret = self.system(cmd) if not ret: self.die("=> Execution of delete commands %s failed\n => see logfile at %s\n\n" % (os.path.abspath(self.log_file_name))) # insert new data self.log("# - Inserting new dynamic data in schema %s" % ex, True) self.execute_sql_file_schema(new_file, ex) self.log("# => %s finished for exercise %s " % (new_file, ex), True) self.log_hline() self.log("# - Remove dynamic data files for exercise %s" % ex, True) os.remove(old_file) os.remove(new_file) self.log_hline() self.log("#### END ####", True) self.log_hline() return 0 def log_hline(self): self.log("# -------------------------------------------------------------------------------", True) def log(self, logline, log_to_stdout = False): if self.log_file_name == None: print("Logfile name not yet set - log: %s" % logline, file=sys.stderr) return with open( self.log_file_name, 'a' ) as f: f.write(logline) f.write("\n") if log_to_stdout: print(logline) def die( self, message ): self.log( message ) print("fatal error: %s" % message, file=sys.stderr) sys.exit( 3 ) def system( self, cmdstring, show_output = False ): self.log( "# Executing '%s'" % cmdstring, False) if hasattr(sys, 'getwindowsversion'): p = subprocess.Popen( cmdstring ) ret = p.wait() else: p = subprocess.Popen( cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) ret = p.wait() if show_output: error = p.stdout.read() print(error, file=sys.stderr) if ( ret != 0 ): print("Error executing %s" % cmdstring, file=sys.stderr) return ret == 0 def execute_sql_file(self, f): self.log( "Run sql-file %s " % f, True) ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, self.log_file_name)) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name))) def execute_sql_file_schema(self, f, schema): self.log( "Run sql-file %s " % f, True) cmd = '%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, schema, self.log_file_name) self.log(" Running command %s" % cmd, True) ret = self.system(cmd) if not ret: self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name))) def dump_dynamic_tables(self, schema, tables): self.log("# - Dump dynamic tables in schema %s" % schema, True) # cmd = '%s -h %s -U %s -t %srec_dynamic_participant_data -t %seem_event -t %srec_dus_mine_state -t %srec_equipment_gear_charge_state -t %srec_history_state --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+".", schema+".", schema+".", schema+".", schema+".", self.database, schema+".sql") for tabl in tables: cmd = '%s -h %s -U %s -t %s --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+"."+tabl, self.database, schema+"_"+tabl+".sql") self.log(" Running command %s" % cmd, True) ret = self.system(cmd) if not ret: self.die("=> Execution of pg_dump failed\n => see logfile at %s\n\n" % os.path.abspath(self.log_file_name)) def read(self, filename): """Read a file and return a list of the lines.""" with open(filename, 'r') as f: lines = f.read().split('\n') return lines def write(self, filename, lines): """Write a file for a given list of lines.""" with open(filename, 'w') as f: for line in lines: f.write(line + '\n') def loadChargeTypes(self): """Read old and new charge types from a .csv file to a dictionary { old_id : new_id }.""" chargeTypes = {} with open(self.csv_file, 'r') as f: lines = f.read().split('\n') for line in lines[1:]: line = line.replace('\r', '') try: old, new = line.split(';') if old in chargeTypes.keys(): print("duplicate old charge type &s".arg(old)) continue chargeTypes[old] = new except: pass return chargeTypes def replaceChargeTypes(self,lines): """Replace all 'chargeTypeId' entries with new ones.""" def replace(iLine, line): if "chargetypeid" in line.lower(): # ignore lines without charge type ID items = line.split() # get all items in line separated by whitespace for i, item in enumerate(map(lambda x: x.lower(), items)): # .lower() to catch all cases if "chargetypeid" in item: oldId = items[i+1].replace(',', '').replace('}', '').replace(']', '').replace("'", "") # remove characters to obtain ID try: newId = self.chargeTypes[oldId] # try lookup in chargeTypes items[i+1] = items[i+1].replace(oldId, newId) # replace old ID with new ID except: if int(oldId) < -2: # any ID < -2 becomes 0 items[i+1] = items[i+1].replace(oldId, '0') else: # -1 and -2 are valid IDs pass line = ' '.join(items) # rejoin items to new line return line return [replace(i, line) for i, line in enumerate(lines)] # simple list comprehension to get file content with updated IDs def main(): p = main_class() p.initialize() p.execute() sys.exit(0) if __name__ == "__main__": main()