229 lines
9.2 KiB
Python
Executable File
229 lines
9.2 KiB
Python
Executable File
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# $Id$
|
|
# $HeadURL$
|
|
# $Author$
|
|
#
|
|
|
|
from __future__ import print_function
|
|
from collections import defaultdict
|
|
|
|
import datetime
|
|
import subprocess
|
|
import sys
|
|
import os
|
|
|
|
class main_class(object):
|
|
def __init__( self ):
|
|
self.project = "guez"
|
|
self.database = "lts_ets_pro"
|
|
self.db_host = "localhost"
|
|
self.db_user = "postgres"
|
|
self.log_dir = '.'
|
|
self.log_file_name = "%s/update_db_%s.log" % (self.log_dir, self.project)
|
|
self.psql_command = 'psql'
|
|
self.pg_dump_command = 'pg_dump'
|
|
self.csv_file = 'guez_db_update_charge_types.csv'
|
|
self.chargeTypes = {}
|
|
|
|
def initialize(self):
|
|
|
|
self.log_hline()
|
|
self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True)
|
|
self.log("# DATE : %s" % datetime.datetime.today(), True)
|
|
self.log("# PROJECT : %s" % self.project, True)
|
|
self.log("# DATABASE : %s" % self.database, True)
|
|
self.log_hline()
|
|
|
|
def execute(self):
|
|
|
|
# execute sql files
|
|
self.log("# - Executings sql files on database %s ..." % self.database, True)
|
|
self.log_hline()
|
|
|
|
f = 'guez_db_update_charge_types_before.sql'
|
|
self.log("# - Executing sql file %s ..." % f, True)
|
|
self.execute_sql_file(f)
|
|
self.log("# => %s sql finished" % f, True)
|
|
self.log_hline()
|
|
|
|
self.log("# - Load charge types csv file %s" % self.csv_file, True)
|
|
self.chargeTypes = self.loadChargeTypes()
|
|
self.log_hline()
|
|
|
|
f = 'guez_db_create_charge_types_foreign_keys.sql'
|
|
exercises = ['ex_14', 'ex_17', 'ex_21', 'ex_22', 'ex_26', 'ex_27', 'ex_28', 'ex_29', 'ex_30', 'ex_31', 'ex_32', 'ex_33', 'ex_34', 'ex_35', 'ex_36', 'ex_37', 'ex_38', 'ex_39', 'ex_40', 'ex_41', 'ex_42', 'ex_44', 'ex_45', 'ex_46', 'ex_51']
|
|
tables = ['eem_event', 'rec_dynamic_participant_data', 'rec_history_state', 'rec_dus_mine_state', 'rec_equipment_gear_charge_state']
|
|
for ex in exercises:
|
|
|
|
self.dump_dynamic_tables(ex, tables)
|
|
|
|
self.log("# - Creating foreign keys (sql file %s) in schema %s" % (f, ex), True)
|
|
self.execute_sql_file_schema(f, ex)
|
|
self.log("# => %s finished for exercise %s " % (f, ex), True)
|
|
self.log_hline()
|
|
|
|
f = 'guez_db_update_charge_types_after.sql'
|
|
self.log("# - UPDATING ids - executing sql file %s ..." % f, True)
|
|
self.execute_sql_file(f)
|
|
self.log("# => %s sql finished" % f, True)
|
|
self.log_hline()
|
|
|
|
for ex in exercises:
|
|
for tabl in tables:
|
|
# replace charge types
|
|
old_file = ex+"_"+tabl+".sql"
|
|
new_file = ex+"_"+tabl+"_new.sql"
|
|
self.log("# - Replace charge types in file %s" % old_file, True)
|
|
db = self.read(old_file)
|
|
db = self.replaceChargeTypes(db)
|
|
self.write(new_file, db)
|
|
|
|
# delete from tables
|
|
self.log("# - Deleting dynamic data in schema %s" % ex, True)
|
|
cmd = '%s -h %s -U %s -c "DELETE FROM %s;" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, tabl, self.database, ex, self.log_file_name)
|
|
self.log(" Running command %s" % cmd, True)
|
|
ret = self.system(cmd)
|
|
if not ret:
|
|
self.die("=> Execution of delete commands %s failed\n => see logfile at %s\n\n" % (os.path.abspath(self.log_file_name)))
|
|
|
|
# insert new data
|
|
self.log("# - Inserting new dynamic data in schema %s" % ex, True)
|
|
self.execute_sql_file_schema(new_file, ex)
|
|
self.log("# => %s finished for exercise %s " % (new_file, ex), True)
|
|
self.log_hline()
|
|
|
|
self.log("# - Remove dynamic data files for exercise %s" % ex, True)
|
|
os.remove(old_file)
|
|
os.remove(new_file)
|
|
|
|
self.log_hline()
|
|
self.log("#### END ####", True)
|
|
self.log_hline()
|
|
|
|
return 0
|
|
|
|
def log_hline(self):
|
|
self.log("# -------------------------------------------------------------------------------", True)
|
|
|
|
def log(self, logline, log_to_stdout = False):
|
|
if self.log_file_name == None:
|
|
print("Logfile name not yet set - log: %s" % logline, file=sys.stderr)
|
|
return
|
|
|
|
with open( self.log_file_name, 'a' ) as f:
|
|
f.write(logline)
|
|
f.write("\n")
|
|
if log_to_stdout:
|
|
print(logline)
|
|
|
|
def die( self, message ):
|
|
self.log( message )
|
|
print("fatal error: %s" % message, file=sys.stderr)
|
|
sys.exit( 3 )
|
|
|
|
def system( self, cmdstring, show_output = False ):
|
|
self.log( "# Executing '%s'" % cmdstring, False)
|
|
if hasattr(sys, 'getwindowsversion'):
|
|
p = subprocess.Popen( cmdstring )
|
|
ret = p.wait()
|
|
else:
|
|
p = subprocess.Popen( cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
|
|
ret = p.wait()
|
|
if show_output:
|
|
error = p.stdout.read()
|
|
print(error, file=sys.stderr)
|
|
if ( ret != 0 ):
|
|
print("Error executing %s" % cmdstring, file=sys.stderr)
|
|
return ret == 0
|
|
|
|
def execute_sql_file(self, f):
|
|
self.log( "Run sql-file %s " % f, True)
|
|
ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, self.log_file_name))
|
|
if not ret:
|
|
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name)))
|
|
|
|
def execute_sql_file_schema(self, f, schema):
|
|
self.log( "Run sql-file %s " % f, True)
|
|
cmd = '%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, schema, self.log_file_name)
|
|
self.log(" Running command %s" % cmd, True)
|
|
ret = self.system(cmd)
|
|
if not ret:
|
|
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name)))
|
|
|
|
def dump_dynamic_tables(self, schema, tables):
|
|
self.log("# - Dump dynamic tables in schema %s" % schema, True)
|
|
|
|
# cmd = '%s -h %s -U %s -t %srec_dynamic_participant_data -t %seem_event -t %srec_dus_mine_state -t %srec_equipment_gear_charge_state -t %srec_history_state --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+".", schema+".", schema+".", schema+".", schema+".", self.database, schema+".sql")
|
|
|
|
for tabl in tables:
|
|
cmd = '%s -h %s -U %s -t %s --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+"."+tabl, self.database, schema+"_"+tabl+".sql")
|
|
self.log(" Running command %s" % cmd, True)
|
|
ret = self.system(cmd)
|
|
if not ret:
|
|
self.die("=> Execution of pg_dump failed\n => see logfile at %s\n\n" % os.path.abspath(self.log_file_name))
|
|
|
|
def read(self, filename):
|
|
"""Read a file and return a list of the lines."""
|
|
with open(filename, 'r') as f:
|
|
lines = f.read().split('\n')
|
|
return lines
|
|
|
|
def write(self, filename, lines):
|
|
"""Write a file for a given list of lines."""
|
|
with open(filename, 'w') as f:
|
|
for line in lines:
|
|
f.write(line + '\n')
|
|
|
|
def loadChargeTypes(self):
|
|
"""Read old and new charge types from a .csv file to a dictionary { old_id : new_id }."""
|
|
chargeTypes = {}
|
|
with open(self.csv_file, 'r') as f:
|
|
lines = f.read().split('\n')
|
|
for line in lines[1:]:
|
|
line = line.replace('\r', '')
|
|
try:
|
|
old, new = line.split(';')
|
|
|
|
if old in chargeTypes.keys():
|
|
print("duplicate old charge type &s".arg(old))
|
|
continue
|
|
|
|
chargeTypes[old] = new
|
|
except:
|
|
pass
|
|
return chargeTypes
|
|
|
|
def replaceChargeTypes(self,lines):
|
|
"""Replace all 'chargeTypeId' entries with new ones."""
|
|
def replace(iLine, line):
|
|
if "chargetypeid" in line.lower(): # ignore lines without charge type ID
|
|
items = line.split() # get all items in line separated by whitespace
|
|
for i, item in enumerate(map(lambda x: x.lower(), items)): # .lower() to catch all cases
|
|
if "chargetypeid" in item:
|
|
oldId = items[i+1].replace(',', '').replace('}', '').replace(']', '').replace("'", "") # remove characters to obtain ID
|
|
try:
|
|
newId = self.chargeTypes[oldId] # try lookup in chargeTypes
|
|
items[i+1] = items[i+1].replace(oldId, newId) # replace old ID with new ID
|
|
except:
|
|
if int(oldId) < -2: # any ID < -2 becomes 0
|
|
items[i+1] = items[i+1].replace(oldId, '0')
|
|
else: # -1 and -2 are valid IDs
|
|
pass
|
|
line = ' '.join(items) # rejoin items to new line
|
|
return line
|
|
|
|
return [replace(i, line) for i, line in enumerate(lines)] # simple list comprehension to get file content with updated IDs
|
|
|
|
def main():
|
|
|
|
p = main_class()
|
|
p.initialize()
|
|
p.execute()
|
|
|
|
sys.exit(0)
|
|
|
|
if __name__ == "__main__":
|
|
main()
|