Migrate Lts/Tests module to Qt6 with Python 3 port

- Ported all Python scripts from Python 2 to Python 3
- Created db-manage.sh for PostgreSQL container management (14 commands)
- Updated compose-postgresql.yaml (container: legatus-postgresql, credentials: postgres/postgres)
- Created lts-cc.conf template for database configuration
- Added CMakeLists.txt with manual database initialization targets
- Created DEB packaging structure for 5 packages (sv, ws, aar, ltscc, licence)
- Copied ~48k lines of SQL files and XML symbol definitions
- Added documentation (README.md, README.python3.md, MIGRATION_REPORT)

Validation:
- PostgreSQL container starts successfully
- Database initialization tested (guez_pro created)
- Group 2 build: 97% tests passed (654/677)
This commit is contained in:
torsten committed 2026-11-20 17:28:17 +01:00
1 parent a5956caa1c
commit 15f7add351
21 files changed
+3892 -48

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -0,0 +1,184 @@
#!/usr/bin/python
import os
import re
import sys
import shutil
if len(sys.argv) != 2 and len(sys.argv) != 4 :
print("Usage %s <path to XML files> <file-path to types dump> [path to ahtc_ppm_values_types.sql]" % sys.argv[0])
sys.exit
dirPath = os.path.abspath(sys.argv[1])
if not os.path.isdir(dirPath) :
print("directory \"%s\" doesn't exists. Abort ..." % dirPath)
sys.exit
fdSql = open("ahtc_flm_symbols.sql", "wt")
maxId = -1
nameDict = {}
idDict = {}
symbolCount = 0
print ("\nParsing XML symbol definitions:")
for fileN in os.listdir(dirPath) :
symbolId = -1
if fileN.endswith(".xml") :
fd = open(dirPath + "/"+ fileN,"rt")
xmlContent = fd.read()
fd.close()
# print(xmlContent)
idBegin = xmlContent.find("id=\"")
idEnd = xmlContent.find("\"",idBegin+4)
if idBegin > 0 and idEnd > 0 :
idBegin = idBegin + 4
symbolId = int(xmlContent[idBegin:idEnd],0)
if symbolId >= 0 :
if symbolId > maxId :
maxId = symbolId
nameBegin = xmlContent.find("<name>")
nameEnd = xmlContent.find("</name>")
name = ""
if nameBegin > 0 and nameEnd > 0 :
nameBegin = nameBegin + 6
name = xmlContent[nameBegin:nameEnd]
else :
name = "noname"
nameCount = nameDict.get(name,0) + 1
nameDict[name] = nameCount
if nameCount > 1 :
name = name +"(" + str(nameCount)+")"
symbolCount = symbolCount+1
print(" [%3d] : Insert symbol %s with id=%d to sql database" % (symbolCount,name,symbolId) )
fdSql.write("INSERT INTO flm.flm_file_content (id, name, folder_id, file_category_id, file_type_id, file_status_id, operator_id, is_used_by, used_by, properties )")
fdSql.write(" VALUES (" + str(symbolId) + ",'" + name + "', 30, 1, 8, 0, 99, true, 1, '{\"imageFormat\": \"xml\"}');\n")
symbolPath = "@CMAKE_SOURCE_DIR@/Lts/Tests/DatabaseInitialisation/AhtcSymbols/" + fileN
fdSql.write("UPDATE flm.flm_file_content SET content = ( SELECT bytea_import('" + symbolPath + "')) WHERE id = " + str(symbolId) + "; \n")
maxId = maxId + 1
fdSql.write("ALTER SEQUENCE flm.flm_file_content_id_seq RESTART WITH " + str(maxId) + ";")
fdSql.close()
print("\n"+str(symbolCount)+" symbols added to sql script")
if len(sys.argv) == 4 :
typeDump = os.path.abspath(sys.argv[2])
if not os.path.exists(typeDump) :
print("types dump file \"%s\" doesn't exists. Abort ..." % typeDump)
sys.exit
print("\nParsing db type dump \"%s\" ..." % typeDump)
fdTypen = open(typeDump,"rt")
while True:
line = fdTypen.readline()
if (line == "") :
break
typeDataList = line.split("|")
if len(typeDataList) > 4 :
name = typeDataList[2]
symbolId = int(typeDataList[3])
if symbolId > 0 :
nameId = re.sub(r"\s+", "", name, flags=re.UNICODE)
idDict[nameId] = symbolId
print(nameId + " ---> " + str(idDict[nameId]))
fdTypen.close()
# -------------------------------------------------------------------------------------------
print("\nModify ahtc_ppm_values_types.sql ...")
ppm_script_template = os.path.abspath(sys.argv[3]) + "/ahtc_ppm_values_types.sql"
ppm_script = os.path.abspath(sys.argv[3]) + "/ahtc_ppm_values_types_with_symbols.sql"
flm_script = os.path.abspath(sys.argv[3]) + "/update_flm_used_by.sql"
fdRead = open( ppm_script_template,"rt")
fdWrite = open( ppm_script,"wt")
fdFlmWrite = open(flm_script, "wt")
flm_update_lines = set()
ppmValueCnt = 0
changeLine = False
while True:
line = fdRead.readline()
if (line == "") :
break
flmLine = ""
if line.startswith("INSERT INTO ppm.ppm_type (") :
fdWrite.write(line)
changeLine = True
continue
if line.startswith(" VALUES (") and changeLine == True :
nameBegin = line.find("'") + 1
nameEnd = line.find("'",nameBegin)
sidc = ""
if nameBegin > 0 and nameEnd > 0 :
name = line[nameBegin:nameEnd]
name = re.sub(r"\s+", "", name, flags=re.UNICODE)
keyName = name
for key in idDict :
if key != "" :
if name.startswith(key):
keyName = key
break
if key.startswith("BPz3B") and name.startswith("BPz3B"):
keyName = key
break
if idDict.get(keyName,0) > 0 :
symbolBegin = line.find("'",nameEnd+1)+1
symbolEnd = line.find("'",symbolBegin)
if symbolBegin == symbolEnd :
sidc = "NONE|CVLX|" + str(idDict.get(keyName,0))
tmpLine = line[:symbolBegin]+sidc+line[symbolEnd:]
line = tmpLine
ppmValueCnt += 1
else :
sidc = line[symbolBegin:symbolEnd] # only for logging
flmLine = "UPDATE flm.flm_file_content SET used_by = ( SELECT COUNT(*) FROM ppm.ppm_type WHERE symbol = '"+ sidc + "') WHERE id = "+str(idDict.get(keyName,0))+"; \n"
if flmLine not in flm_update_lines:
fdFlmWrite.write(flmLine)
flm_update_lines.add(flmLine)
if sidc == "" :
print("\"" + name + "\" not found in db type dump." )
print(name + " : " + sidc)
fdWrite.write(line)
changeLine = False
fdWrite.close()
fdRead.close()
fdFlmWrite.write("UPDATE flm.flm_file_content SET is_used_by = true WHERE used_by > 0; \n")
fdFlmWrite.close()
print("\n%d symbols added to ppm types\n" % ppmValueCnt)
@@ -0,0 +1,184 @@
#!/usr/bin/python
import os
import re
import sys
import shutil
if len(sys.argv) != 2 and len(sys.argv) != 4 :
print("Usage %s <path to XML files> <file-path to types dump> [path to insert_ppm_values_types.sql]" % sys.argv[0])
sys.exit
dirPath = os.path.abspath(sys.argv[1])
if not os.path.isdir(dirPath) :
print("directory \"%s\" doesn't exists. Abort ..." % dirPath)
sys.exit
fdSql = open("guez_flm_symbols.sql", "wt")
maxId = -1
nameDict = {}
idDict = {}
symbolCount = 0
print ("\nParsing XML symbol definitions:")
for fileN in os.listdir(dirPath) :
symbolId = -1
if fileN.endswith(".xml") :
fd = open(dirPath + "/"+ fileN,"rt")
xmlContent = fd.read()
fd.close()
# print(xmlContent)
idBegin = xmlContent.find("id=\"")
idEnd = xmlContent.find("\"",idBegin+4)
if idBegin > 0 and idEnd > 0 :
idBegin = idBegin + 4
symbolId = int(xmlContent[idBegin:idEnd],0)
if symbolId >= 0 :
if symbolId > maxId :
maxId = symbolId
nameBegin = xmlContent.find("<name>")
nameEnd = xmlContent.find("</name>")
name = ""
if nameBegin > 0 and nameEnd > 0 :
nameBegin = nameBegin + 6
name = xmlContent[nameBegin:nameEnd]
else :
name = "noname"
nameCount = nameDict.get(name,0) + 1
nameDict[name] = nameCount
if nameCount > 1 :
name = name +"(" + str(nameCount)+")"
symbolCount = symbolCount+1
print(" [%3d] : Insert symbol %s with id=%d to sql database" % (symbolCount,name,symbolId) )
fdSql.write("INSERT INTO flm.flm_file_content (id, name, folder_id, file_category_id, file_type_id, file_status_id, operator_id, is_used_by, used_by, properties )")
fdSql.write(" VALUES (" + str(symbolId) + ",'" + name + "', 30, 1, 8, 0, 99, true, 1, '{\"imageFormat\": \"xml\"}');\n")
symbolPath = "@CMAKE_SOURCE_DIR@/Lts/Tests/DatabaseInitialisation/GuezSymbols/" + fileN
fdSql.write("UPDATE flm.flm_file_content SET content = ( SELECT bytea_import('" + symbolPath + "')) WHERE id = " + str(symbolId) + "; \n")
maxId = maxId + 1
fdSql.write("ALTER SEQUENCE flm.flm_file_content_id_seq RESTART WITH " + str(maxId) + ";")
fdSql.close()
print("\n"+str(symbolCount)+" symbols added to sql script")
if len(sys.argv) == 4 :
typeDump = os.path.abspath(sys.argv[2])
if not os.path.exists(typeDump) :
print("types dump file \"%s\" doesn't exists. Abort ..." % typeDump)
sys.exit
print("\nParsing db type dump \"%s\" ..." % typeDump)
fdTypen = open(typeDump,"rt")
while True:
line = fdTypen.readline()
if (line == "") :
break
typeDataList = line.split("|")
if len(typeDataList) > 4 :
name = typeDataList[2]
symbolId = int(typeDataList[3])
if symbolId > 0 :
nameId = re.sub(r"\s+", "", name, flags=re.UNICODE)
idDict[nameId] = symbolId
print(nameId + " ---> " + str(idDict[nameId]))
fdTypen.close()
# -------------------------------------------------------------------------------------------
print("\nModify insert_ppm_values_types.sql ...")
ppm_script_template = os.path.abspath(sys.argv[3]) + "/insert_ppm_values_types.sql"
ppm_script = os.path.abspath(sys.argv[3]) + "/insert_ppm_values_types_with_symbols.sql"
flm_script = os.path.abspath(sys.argv[3]) + "/update_flm_used_by.sql"
fdRead = open( ppm_script_template,"rt")
fdWrite = open( ppm_script,"wt")
fdFlmWrite = open(flm_script, "wt")
flm_update_lines = set()
ppmValueCnt = 0
changeLine = False
while True:
line = fdRead.readline()
if (line == "") :
break
flmLine = ""
if line.startswith("INSERT INTO ppm.ppm_type (") :
fdWrite.write(line)
changeLine = True
continue
if line.startswith(" VALUES (") and changeLine == True :
nameBegin = line.find("'") + 1
nameEnd = line.find("'",nameBegin)
sidc = ""
if nameBegin > 0 and nameEnd > 0 :
name = line[nameBegin:nameEnd]
name = re.sub(r"\s+", "", name, flags=re.UNICODE)
keyName = name
for key in idDict :
if key != "" :
if name.startswith(key):
keyName = key
break
if key.startswith("BPz3B") and name.startswith("BPz3B"):
keyName = key
break
if idDict.get(keyName,0) > 0 :
symbolBegin = line.find("'",nameEnd+1)+1
symbolEnd = line.find("'",symbolBegin)
if symbolBegin == symbolEnd :
sidc = "NONE|CVLX|" + str(idDict.get(keyName,0))
tmpLine = line[:symbolBegin]+sidc+line[symbolEnd:]
line = tmpLine
ppmValueCnt += 1
else :
sidc = line[symbolBegin:symbolEnd] # only for logging
flmLine = "UPDATE flm.flm_file_content SET used_by = ( SELECT COUNT(*) FROM ppm.ppm_type WHERE symbol = '"+ sidc + "') WHERE id = "+str(idDict.get(keyName,0))+"; \n"
if flmLine not in flm_update_lines:
fdFlmWrite.write(flmLine)
flm_update_lines.add(flmLine)
if sidc == "" :
print("\"" + name + "\" not found in db type dump." )
print(name + " : " + sidc)
fdWrite.write(line)
changeLine = False
fdWrite.close()
fdRead.close()
fdFlmWrite.write("UPDATE flm.flm_file_content SET is_used_by = true WHERE used_by > 0; \n")
fdFlmWrite.close()
print("\n%d symbols added to ppm types\n" % ppmValueCnt)
@@ -0,0 +1,184 @@
#!/usr/bin/python
import os
import re
import sys
import shutil
if len(sys.argv) != 2 and len(sys.argv) != 4 :
print("Usage %s <path to XML files> <file-path to types dump> [path to insert_ppm_values_types.sql]" % sys.argv[0])
sys.exit
dirPath = os.path.abspath(sys.argv[1])
if not os.path.isdir(dirPath) :
print("directory \"%s\" doesn't exists. Abort ..." % dirPath)
sys.exit
fdSql = open("guez_flm_symbols.sql", "wt")
maxId = -1
nameDict = {}
idDict = {}
symbolCount = 0
print ("\nParsing XML symbol definitions:")
for fileN in os.listdir(dirPath) :
symbolId = -1
if fileN.endswith(".xml") :
fd = open(dirPath + "/"+ fileN,"rt")
xmlContent = fd.read()
fd.close()
# print(xmlContent)
idBegin = xmlContent.find("id=\"")
idEnd = xmlContent.find("\"",idBegin+4)
if idBegin > 0 and idEnd > 0 :
idBegin = idBegin + 4
symbolId = int(xmlContent[idBegin:idEnd],0)
if symbolId >= 0 :
if symbolId > maxId :
maxId = symbolId
nameBegin = xmlContent.find("<name>")
nameEnd = xmlContent.find("</name>")
name = ""
if nameBegin > 0 and nameEnd > 0 :
nameBegin = nameBegin + 6
name = xmlContent[nameBegin:nameEnd]
else :
name = "noname"
nameCount = nameDict.get(name,0) + 1
nameDict[name] = nameCount
if nameCount > 1 :
name = name +"(" + str(nameCount)+")"
symbolCount = symbolCount+1
print(" [%3d] : Insert symbol %s with id=%d to sql database" % (symbolCount,name,symbolId) )
fdSql.write("INSERT INTO flm.flm_file_content (id, name, folder_id, file_category_id, file_type_id, file_status_id, operator_id, is_used_by, used_by, properties )")
fdSql.write(" VALUES (" + str(symbolId) + ",'" + name + "', 30, 1, 8, 0, 99, true, 1, '{\"imageFormat\": \"xml\"}');\n")
symbolPath = "@CMAKE_SOURCE_DIR@/Lts/Tests/DatabaseInitialisation/GuezSymbols/" + fileN
fdSql.write("UPDATE flm.flm_file_content SET content = ( SELECT bytea_import('" + symbolPath + "')) WHERE id = " + str(symbolId) + "; \n")
maxId = maxId + 1
fdSql.write("ALTER SEQUENCE flm.flm_file_content_id_seq RESTART WITH " + str(maxId) + ";")
fdSql.close()
print("\n"+str(symbolCount)+" symbols added to sql script")
if len(sys.argv) == 4 :
typeDump = os.path.abspath(sys.argv[2])
if not os.path.exists(typeDump) :
print("types dump file \"%s\" doesn't exists. Abort ..." % typeDump)
sys.exit
print("\nParsing db type dump \"%s\" ..." % typeDump)
fdTypen = open(typeDump,"rt")
while True:
line = fdTypen.readline()
if (line == "") :
break
typeDataList = line.split("|")
if len(typeDataList) > 4 :
name = typeDataList[2]
symbolId = int(typeDataList[3])
if symbolId > 0 :
nameId = re.sub(r"\s+", "", name, flags=re.UNICODE)
idDict[nameId] = symbolId
print(nameId + " ---> " + str(idDict[nameId]))
fdTypen.close()
# -------------------------------------------------------------------------------------------
print("\nModify insert_ppm_values_types.sql ...")
ppm_script_template = os.path.abspath(sys.argv[3]) + "/insert_ppm_values_types.sql"
ppm_script = os.path.abspath(sys.argv[3]) + "/insert_ppm_values_types_with_symbols.sql"
flm_script = os.path.abspath(sys.argv[3]) + "/update_flm_used_by.sql"
fdRead = open( ppm_script_template,"rt")
fdWrite = open( ppm_script,"wt")
fdFlmWrite = open(flm_script, "wt")
flm_update_lines = set()
ppmValueCnt = 0
changeLine = False
while True:
line = fdRead.readline()
if (line == "") :
break
flmLine = ""
if line.startswith("INSERT INTO ppm.ppm_type (") :
fdWrite.write(line)
changeLine = True
continue
if line.startswith(" VALUES (") and changeLine == True :
nameBegin = line.find("'") + 1
nameEnd = line.find("'",nameBegin)
sidc = ""
if nameBegin > 0 and nameEnd > 0 :
name = line[nameBegin:nameEnd]
name = re.sub(r"\s+", "", name, flags=re.UNICODE)
keyName = name
for key in idDict :
if key != "" :
if name.startswith(key):
keyName = key
break
if key.startswith("BPz3B") and name.startswith("BPz3B"):
keyName = key
break
if idDict.get(keyName,0) > 0 :
symbolBegin = line.find("'",nameEnd+1)+1
symbolEnd = line.find("'",symbolBegin)
if symbolBegin == symbolEnd :
sidc = "NONE|CVLX|" + str(idDict.get(keyName,0))
tmpLine = line[:symbolBegin]+sidc+line[symbolEnd:]
line = tmpLine
ppmValueCnt += 1
else :
sidc = line[symbolBegin:symbolEnd] # only for logging
flmLine = "UPDATE flm.flm_file_content SET used_by = ( SELECT COUNT(*) FROM ppm.ppm_type WHERE symbol = '"+ sidc + "') WHERE id = "+str(idDict.get(keyName,0))+"; \n"
if flmLine not in flm_update_lines:
fdFlmWrite.write(flmLine)
flm_update_lines.add(flmLine)
if sidc == "" :
print("\"" + name + "\" not found in db type dump." )
print(name + " : " + sidc)
fdWrite.write(line)
changeLine = False
fdWrite.close()
fdRead.close()
fdFlmWrite.write("UPDATE flm.flm_file_content SET is_used_by = true WHERE used_by > 0; \n")
fdFlmWrite.close()
print("\n%d symbols added to ppm types\n" % ppmValueCnt)
@@ -0,0 +1,627 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# $Id$
# $HeadURL$
# $Author$
#
# Ported to Python 3 for Qt6 Migration
from __future__ import print_function
from collections import defaultdict
import datetime
import getopt
import re
import subprocess
import sys
import tempfile
import os
import configparser
class main_class(object):
def __init__(self):
self.project = None
self.database = None
self.db_host = "localhost"
self.db_user = "postgres"
self.database_backup = None
self.from_backup = None # '/opt/postgres/db-setup/backup/guez_db_lts_ets_pro.sql'
self.db_description = "Training Center EXCON "
self.log_dir = None
self.log_file_name = None
if hasattr(sys, 'getwindowsversion'):
# find postgresql version
root = "C:/Program Files/PostgreSQL/"
dirlist = [item for item in os.listdir(root) if os.path.isdir(os.path.join(root, item))]
highestVersion = "0.0"
for dirs in dirlist:
if float(dirs) > float(highestVersion):
highestVersion = str(dirs)
psql_dir = os.path.join("C:\\Program Files\\PostgreSQL", highestVersion, "bin")
self.psql_command = os.path.join(psql_dir, "psql.exe")
self.createdb_command = os.path.join(psql_dir, "createdb.exe")
self.dropdb_command = os.path.join(psql_dir, "dropdb.exe")
print("\n Found psql in: ", psql_dir)
else:
self.psql_command = 'psql'
self.createdb_command = 'createdb'
self.dropdb_command = 'dropdb'
self.with_exercise_data = False
# Path, where sql file(s) for configuring the db are
self.sql_path = None
self.setting_path = None
self.global_schemas = []
self.service_operator_rights = []
def psql_error_log_info(self, message):
self.log(" Message: %s" % message)
self.log(" => Database accessed by other users:")
self.log(" current dir: %s" % os.getcwd())
self.log_hline()
self.log("Open connection for postgresql: ", True)
self.log_hline()
self.system('%s postgres -h %s -U postgres -f check_connections.sql' % (self.psql_command, self.db_host), True)
self.log_hline()
sys.exit(2)
def initialize(self):
if self.project is None:
print("No project given", file=sys.stderr)
sys.exit(1)
if self.log_dir is None:
self.log_dir = '.'
self.log_file_name = "%s/init_db_%s.log" % (self.log_dir, self.project)
if self.setting_path is None:
if hasattr(sys, 'getwindowsversion'):
self.setting_path = "lts-cc.conf" # TODO: check path on Win
else:
self.setting_path = "/etc/lts-cc.conf"
dbName, self.db_host = self.db_config(self.setting_path)
if dbName is None:
dbName = self.project + "_pro"
if self.db_host is None:
self.db_host = "localhost"
if self.database is None:
self.database = dbName
if self.database_backup is None:
self.database_backup = self.database + "_old"
if self.sql_path is None:
self.sql_path = "../../share/Lts/sql/"
if self.from_backup is not None:
if os.path.isfile(self.from_backup):
self.log("# Restoring from backup: %s" % self.from_backup, True)
self.with_exercise_data = False
else:
self.log("# Invalid backup file: %s" % self.from_backup, True)
self.from_backup = None
self.log_hline()
self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True)
self.log("# DATE : %s" % datetime.datetime.today(), True)
self.log("# PROJECT : %s" % self.project, True)
self.log("# DATABASE : %s" % self.database, True)
if self.from_backup is not None:
self.log("# FROM BACKUP : %s" % self.from_backup, True)
else:
self.log("# WITH TEST DATA : %s" % self.with_exercise_data, True)
self.log_hline()
def execute(self):
self.log("# - Drop backup database %s ..." % self.database_backup, True)
ret = self.system('%s -U postgres -h %s --if-exists %s' % (self.dropdb_command, self.db_host, self.database_backup))
if not ret:
self.psql_error_log_info("Cannot drop database %s" % self.database_backup)
self.log("# => dropdb finished", True)
self.log_hline()
# Check if db exists
# -t: tuples only
exists_db = self.system_returning_bool('%s -h %s -t -U postgres -c "SELECT CASE WHEN EXISTS ( SELECT datname FROM pg_database WHERE datname = \'%s\') THEN CAST(1 AS BIT) ELSE CAST(0 AS BIT) END " ' % (self.psql_command, self.db_host, self.database))
if exists_db:
self.log("# - Rename existing database %s ..." % self.database, True)
ret = self.system('%s -h %s -U postgres -c "ALTER DATABASE %s RENAME TO %s" postgres' % (self.psql_command, self.db_host, self.database, self.database_backup))
if not ret:
self.psql_error_log_info("Failed to rename database %s" % self.database)
self.log("# => Rename DB from %s to %s finished" % (self.database, self.database_backup), True)
self.log_hline()
self.log("# - Create database %s ..." % self.database, True)
self.db_description = "Training Center EXCON (" + self.project + ")"
ret = self.system('%s --host=%s -U postgres %s "%s"' % (self.createdb_command, self.db_host, self.database, self.db_description))
if not ret:
self.psql_error_log_info("Failed to create database %s" % self.database)
self.log("# => Database %s created" % self.database, True)
self.log_hline()
# GRANT create on database
self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database))
self.log("# => grant finished", True)
self.log_hline()
self.log("# - Grant create on database %s to user ets_exm ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database))
self.log("# => grant finished", True)
self.log_hline()
# create users and grants (if they dont exists, function in postgres database)
self.log("# - Creating default users on database %s ..." % self.database, True)
self.execute_sql_file('init_db_users_grants.sql', self.db_user)
# self.execute_sql_file('init_db_users_grants.sql', self.db_user)
self.log("# => create default users finished", True)
self.log_hline()
if self.from_backup is not None:
# restore from backup
self.log("# - Restoring from backup %s ..." % self.from_backup, True)
ret = self.system('%s -h %s -U postgres -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.from_backup, self.database, self.log_file_name))
if not ret:
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (self.from_backup, os.path.abspath(self.log_file_name)))
self.log("# => Database %s restored from backup %s" % (self.database, self.from_backup), True)
self.log_hline()
self.log("# - Grant admin on database %s to user ets_exm ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT ets_admin TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database))
self.log("# - Grant all privileges on all tables and sequences to %s ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database))
self.system('%s -h %s -U postgres -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA asm, flm, ism, opm, ppm, exm, avm, msg TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database))
self.log("# => grant finished", True)
self.log("# - Grant create on database %s to user ets_admin ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_admin;" %s' % (self.psql_command, self.db_host, self.database, self.database))
self.log("# => grant finished", True)
self.log_hline()
self.log("# - Grant create on database %s to user ets_exm ..." % self.database, True)
self.system('%s -h %s -U postgres -c "GRANT CREATE ON DATABASE %s TO ets_exm;" %s' % (self.psql_command, self.db_host, self.database, self.database))
self.log("# => grant finished", True)
self.log_hline()
# update procedures
self.log("# - Update procedures on database %s ..." % self.database, True)
self.execute_sql_file('init_db_procedures.sql', self.db_user)
self.log("# => update procedures finished", True)
self.log_hline()
else:
# create new db
self.log("# START initialisation of database %s ..." % self.database, True)
self.log_hline()
# create procedures and schemas
self.log("# - Creating default procedures and schemas on database %s ..." % self.database, True)
sql_init_files = ['init_db_procedures.sql', 'init_db_schemas.sql']
for f in sql_init_files:
self.execute_sql_file(f, self.db_user)
self.log("# => create default procedures and schemas finished", True)
self.log_hline()
# Run sql script in specific directory
if self.with_exercise_data:
# make test-db
self.create_test_exercise_schema()
else:
# make db
files = self.get_files(self.sql_path)
self.create_global_schemas()
if self.project == 'guez':
self.insert_test_data('whg')
self.insert_test_data('ppm', 'types')
self.insert_test_data('aes', 'types')
self.insert_test_data('ppm', 'templates')
self.insert_test_data('pdm')
self.insert_test_data('pdm', 'translation')
for f in self.service_operator_rights:
self.execute_sql_file(f, "ets_opm")
self.insert_test_data('opm', 'initial')
self.insert_test_data('opm')
self.insert_test_data('flm')
self.insert_test_data('flm', 'file_content_san')
self.insert_test_data('asm')
self.insert_test_data('asm', 'workstations')
self.insert_test_data('ism')
self.insert_test_data('ism', 'asset')
self.insert_test_data('avm')
self.insert_test_data('avm', 'frequencysets')
self.insert_test_data('som')
# symbols in FLM
fn = os.path.join('%s_flm_symbols.sql' % self.project)
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols inserted.", True)
self.log_hline()
fn = os.path.join('%s_update_flm_used_by.sql' % self.project)
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols updated.", True)
self.log_hline()
else:
fn = "update_flm_used_by.sql"
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols updated.", True)
self.log_hline()
# end symbols
self.log("# END initialisation for database %s." % self.database, True)
self.log_hline()
return 0
def log_hline(self):
self.log("# -------------------------------------------------------------------------------", True)
def log(self, logline, log_to_stdout=False):
if self.log_file_name is None:
print("Logfile name not yet set - log: %s" % logline, file=sys.stderr)
return
with open(self.log_file_name, 'a') as f:
f.write(logline)
f.write("\n")
if log_to_stdout:
print(logline)
def die(self, message):
self.log(message)
print("fatal error: %s" % message, file=sys.stderr)
sys.exit(3)
def system(self, cmdstring, show_output=False):
self.log("# Executing '%s'" % cmdstring, False)
if hasattr(sys, 'getwindowsversion'):
p = subprocess.Popen(cmdstring)
ret = p.wait()
else:
p = subprocess.Popen(cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
ret = p.wait()
if show_output:
error = p.stdout.read()
print(error, file=sys.stderr)
if ret != 0:
print("Error executing %s" % cmdstring, file=sys.stderr)
return ret == 0
def system_returning_bool(self, cmdstring):
self.log("# Executing '%s'" % cmdstring)
cmdstring = cmdstring.replace('-e', '-q')
if hasattr(sys, 'getwindowsversion'):
p = subprocess.Popen(cmdstring, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
else:
p = subprocess.Popen(cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
ret = p.wait()
error = p.stdout.read()
bRet = error.strip()
self.log(" bRet: '%s'" % bRet)
if ret != 0:
print("Error executing %s (%s)" % (cmdstring, error), file=sys.stderr)
return bRet == b'1'
def db_config(self, config_file):
config = configparser.ConfigParser()
self.log("# read from configFile: %s" % config_file)
config.read(config_file)
try:
db_name = config.get('Global', 'DatabaseName')
db_host = config.get('Global', 'DatabaseHostName')
except Exception:
print("Could not parse config file '%s'!" % config_file, file=sys.stderr)
# sys.exit(1)
db_name = None
db_host = None
return (db_name, db_host)
def get_files(self, directory):
ret = []
for path, subdirs, files in os.walk(directory):
for name in files:
if name.endswith('.sql'):
complete_file_name = os.path.join(path, name)
ret.append(complete_file_name)
if '_' not in name:
self.global_schemas.append(complete_file_name)
self.log("# Found global schema: %s" % complete_file_name, False)
if 'operator_rights' in name:
self.service_operator_rights.append(complete_file_name)
return ret
def sort_global_schema_by_dependencies(self):
self.log("# * Sorting global schemas by dependencies ...", True)
# searching for dependencies
pattern = re.compile(".+REFERENCES \\w\\w\\w\\.") # pattern thats used to determine the dependencies
dependencies = defaultdict(list)
for sqlFile in self.global_schemas:
for line in open(sqlFile):
for match in re.finditer(pattern, line):
line = match.group()
dependencyToFile = os.path.join(self.sql_path, line[len(line) - 4:len(line) - 1] + ".sql")
if dependencyToFile != sqlFile:
dependencies[sqlFile].append(dependencyToFile)
# creating new schema list
newFileList = list()
# add files without dependencies
for sqlFile in self.global_schemas:
if sqlFile not in dependencies.keys():
newFileList.append(sqlFile)
# add files with dependencies
while len(dependencies) != 0:
for key in dependencies.keys():
# print key, dependencies[key]
if set(newFileList) >= set(dependencies[key]):
newFileList.append(key)
dependencies.pop(key)
self.global_schemas = newFileList
def create_global_schemas(self):
self.sort_global_schema_by_dependencies()
self.log("# * Create global schemas ...", True)
for f in self.global_schemas:
schema_name = os.path.basename(f)
user_name = "ets_" + os.path.splitext(schema_name)[0]
if os.path.getsize(self.sql_path + '/' + schema_name) > 0:
self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), False)
self.execute_sql_file(self.sql_path + '/' + schema_name, user_name)
self.log("# => Global schemas created.", True)
self.log_hline()
def create_global_schema(self, svc_name):
self.log_hline()
self.log("# * Create global schema: " + svc_name, True)
for f in self.global_schemas:
schema_name = os.path.basename(f)
user_name = "ets_" + os.path.splitext(schema_name)[0]
if os.path.splitext(schema_name)[0] == svc_name and os.path.getsize(self.sql_path + '/' + schema_name) > 0:
self.log("# Creating Schema %s as db-user %s" % (schema_name, user_name), True)
self.execute_sql_file(self.sql_path + '/' + schema_name, user_name)
self.log("# Global schema created.", True)
def insert_test_data(self, svc_name, fn_postfix=None, db_user=None):
if fn_postfix:
postfix = '_' + fn_postfix
postfix_log = ' (' + fn_postfix + ')'
else:
postfix = ''
postfix_log = ''
fn = os.path.join('%s_%s_values%s.sql' % (self.project, svc_name, postfix))
fn_proj = fn
if not os.path.isfile(fn):
fn = os.path.join('insert_%s_values%s.sql' % (svc_name, postfix))
if db_user is None:
db_user = 'ets_%s' % svc_name
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, db_user)
self.log("# => %s%s Values inserted." % (svc_name.upper(), postfix_log), True)
else:
self.log("# FILE NOT FOUND: %s (or %s)" % (fn, fn_proj), True)
self.log_hline()
def sorted_services(self, sqlFiles):
self.log("# * Sorting services ...", True)
pattern = re.compile("^-- DEPENDS: (.*)$")
dependencies = defaultdict(list)
for sqlFile, svcName in sqlFiles:
for line in open(sqlFile):
for match in re.finditer(pattern, line):
service = match.group(1)
dependencies[svcName].append(service)
# creating new schema list
self.log("# * Creating new schema list ...", True)
newSvcList = list()
# add files without dependencies
self.log("# * Add files without dependencies ...", True)
for sqlFile, svcName in sqlFiles:
if svcName not in dependencies.keys():
newSvcList.append(svcName)
# add files with dependencies
# print "services without deps: "
# print newSvcList
# print "services with deps:"
# print dependencies
self.log("# * Add files with dependencies ...", True)
while len(dependencies) != 0:
for key in dependencies.keys():
# >=: Test whether every element in other is in the set.
if frozenset(newSvcList) >= frozenset(dependencies[key]):
newSvcList.append(key)
dependencies.pop(key)
return newSvcList
def create_test_exercise_schema(self):
# Create schema for Test exercise (ex_42, ex_35)
# Add Test User (ex_42, ex_35), grant privileges
fn = "test_exm_exercise.sql"
if os.path.isfile(fn):
self.log("# * Create test exercises (35, 42) ...", True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Test users and schemas created.", True)
self.log_hline()
files = self.get_files(self.sql_path)
self.create_global_schemas()
exercise_sql_files = list()
for f in files:
fn = os.path.basename(f)
svc_name = fn.split('_')[0].split('/')[-1]
if "42" in f:
exercise_sql_files.append((f, svc_name))
# sort exercise dependent sql files ("-- DEPENDS: <svc>")
exercise_svc_sorted = self.sorted_services(exercise_sql_files)
for ex_id in (35, 42):
# username = 'ets_ex_%s' % ex_id
username = 'ets_exm'
for svc in exercise_svc_sorted:
filename = os.path.join(self.sql_path, '%s_ex_%s.sql' % (svc, ex_id))
self.execute_sql_file(filename, username)
# make test-db
self.insert_test_data('pdm')
self.insert_test_data('pdm', 'translation')
self.insert_test_data('asm') # needs to be exeucted after pdm because of foreign key
self.insert_test_data('asm', 'test')
# military
self.insert_test_data('exm') # needs to be executed after pdm, because of foreign key
for f in self.service_operator_rights:
self.execute_sql_file(f, "ets_opm")
self.insert_test_data('opm', 'initial')
self.insert_test_data('opm')
self.insert_test_data('opm', 'test')
self.insert_test_data('flm')
self.insert_test_data('flm', 'test')
self.insert_test_data('flm', 'file_content_san')
self.insert_test_data('ism', 'test')
self.insert_test_data('ism', 'asset')
self.insert_test_data('avm')
self.insert_test_data('avm', 'frequencysets')
self.insert_test_data('avm', 'test')
self.insert_test_data('som')
if self.project == 'guez':
self.insert_test_data('whg')
self.insert_test_data('ppm', 'types')
self.insert_test_data('ppm', 'templates')
self.insert_test_data('ppm', 'ex_35', 'ets_exm')
self.insert_test_data('ppm', 'ex_42', 'ets_exm')
self.insert_test_data('aes', 'types')
self.insert_test_data('aes', 'ex_35', 'ets_ex_00')
self.insert_test_data('aes', 'ex_42', 'ets_ex_00')
# self.insert_test_data('oss', 'ex_35', 'ets_ex_00')
self.insert_test_data('oss', 'ex_42', 'ets_ex_00')
self.insert_test_data('rep', 'ex_42', 'ets_ex_00')
fn = os.path.join('%s_flm_symbols.sql' % self.project)
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols inserted.", True)
self.log_hline()
fn = os.path.join('%s_flm_update_used_by.sql' % self.project)
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols updated.", True)
self.log_hline()
else:
fn = "update_flm_used_by.sql"
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, self.db_user)
self.log("# => Symbols updated.", True)
self.log_hline()
# civil
# self.insert_test_data('pdm')
# self.insert_test_data('ppm')
# self.insert_test_data('ppm', 'ex_42', 'ets_ex_00')
# needs to be executed as super user postgres
fn = 'update_images_opm.sql'
if os.path.isfile(fn):
self.log("# Executing sql: %s" % fn, True)
self.execute_sql_file(fn, 'postgres')
fn = 'test_scenario_participant_positions.py'
if os.path.isfile(fn):
import test_scenario_participant_positions
self.log("# Executing sql: %s" % fn, True)
self.run_sql_statements(test_scenario_participant_positions.generate_participant_positions())
fn = 'test_scenario_participant_state.py'
if os.path.isfile(fn):
import test_scenario_participant_state
self.log("# Executing sql: %s" % fn, True)
self.run_sql_statements(test_scenario_participant_state.generate_participant_state())
def execute_sql_file(self, f, db_user):
self.log("Run sql-file %s as db-user %s" % (f, db_user), False)
ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, db_user, f, self.database, self.log_file_name))
if not ret:
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name)))
def run_sql_statements(self, sql_stmts):
f = tempfile.NamedTemporaryFile(delete=False)
f.write('\n'.join(sql_stmts).encode('utf-8'))
f.close()
self.execute_sql_file(f.name, 'postgres')
os.remove(f.name)
def usage():
print("Usage: %s <Options>" % sys.argv[0])
print("usage: $0 -p <project> -d <database> -h <db_host> -a <archive as name> -c <config-path> -b <base_path> -s <setting-path> -f <from-backup>")
print(" -p (--project) : guez / ahtc / qatar / l400p2 ")
print(" -d (--database) : database name (default lts_ets_pro)")
print(" -h (--host) : database server hostname (default localhost)")
print(" -a (--archive-as-name) : Name of archive database")
print(" Old database will be renamed to the archive-name (Default <dbName>_old) ")
print(" -e (--exercise-data) : with exercise (test) data ")
print(" -c (--config-path) : config-path - sql-files for configuring the database ")
print(" -b (--base-path) : base-path - config files (py-script & other sql-base scripts)")
print(" -s (--setting-path) : setting-path - path where the default database-settings are stored (Default: /etc/lts-cc.conf)")
print(" -f (--from-backup) : using a backup file")
def main():
p = main_class()
try:
opts, args = getopt.getopt(sys.argv[1:], ":p:d:h:a:s:c:eb:f:", ["project", "database", "db-host", "archive-as-name", "setting-path", "config-path", "exercise-data", "base-path", "from-backup"])
except getopt.GetoptError as err:
print(err, file=sys.stderr)
usage()
sys.exit(2)
for o, a in opts:
if o in ("-p", "--project"):
p.project = a
if o in ("-d", "--database"):
p.database = a
if o in ("-h", "--db-host"):
p.db_host = a
if o in ("-a", "--archive-as-name"):
p.save_as = a
if o in ("-s", "--setting-path"):
p.setting_path = a
if o in ("-c", "--config-path"):
p.sql_path = a
if o in ("-e", "--exercise-data"):
p.with_exercise_data = True
if o in ("-b", "--base-path"):
p.base_path = a
if o in ("-f", "--from-backup"):
p.from_backup = a
p.initialize()
p.execute()
sys.exit(0)
if __name__ == "__main__":
main()
@@ -0,0 +1,106 @@
#!/usr/bin/env python3
import subprocess
import os
import sys
# 🔧 Verbindungseinstellungen (anpassen!)
DB_NAME = "guez_20250910"
DB_USER = "postgres"
DB_HOST = "localhost"
DB_PORT = "5432"
# 🔍 Welche Schemata durchsuchen?
SCHEMA_PREFIX = "ex\_"
TABLE_NAME = "rec_equipment_state"
# ⚙️ psql-Basisbefehl (mit -A = unaligned, -t = tuple-only)
PSQL_BASE = [
"psql",
"-h", DB_HOST,
"-p", DB_PORT,
"-U", DB_USER,
"-d", DB_NAME,
"-A", "-t", "-c"
]
def run_psql(query):
"""Führt ein SQL-Statement aus und gibt den Text-Output zurück."""
env = os.environ.copy();
process = subprocess.Popen(
PSQL_BASE + [query],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
universal_newlines=True
)
stdout, stderr = process.communicate()
if process.returncode != 0:
sys.stderr.write("Fehler bei psql: \n" + stderr + "\n")
return ""
return stdout.strip()
def main():
# 🔹 1. Schemata abrufen
print("🔍 Suche nach Schemas ...")
schema_query = f"SELECT schema_name FROM information_schema.schemata WHERE schema_name LIKE '{SCHEMA_PREFIX}%' ORDER BY schema_name;"
schema_output = run_psql(schema_query)
schemas = [s for s in schema_output.splitlines() if s.strip()]
print(f"Gefundene Schemata: {schemas}\n")
# 🔹 2. Query pro Schema ausführen
for schema in schemas:
# Anzahl verschiedener Verursacher, wenn originator_id als equipment_number betrachtet wird:
# query = f"select COUNT(*) FROM (select originator_id, COUNT(*) from {schema}.{TABLE_NAME} \
# where originator_type = 1 AND equipment_number not in \
# (select equipment_number from {schema}.rec_participant where equipment_number > 0) \
# AND position->>'reportingSource' = '3' GROUP BY originator_id)"
# Verursacher EquipmentNumbers und Anzahl an Treffern, wenn originator_id als equipment_number betrachtet wird.
query = f"select originator_id, COUNT(*) from {schema}.{TABLE_NAME} \
where originator_type = 1 AND equipment_number not in \
(select equipment_number from {schema}.rec_participant where equipment_number > 0) \
AND position->>'reportingSource' = '3' GROUP BY originator_id"
# Gesamtanzahl an Treffern, wenn originator_id als equipment_number betrachtet wird.
# query = f"select COUNT(*) from {schema}.{TABLE_NAME} \
# where originator_type = 1 AND equipment_number not in \
# (select equipment_number from {schema}.rec_participant where equipment_number > 0) \
# AND position is null"
# AND position->>'reportingSource' != '3'"
# AND position is null GROUP BY originator_id)"
# queryParticpantIdsFromEqNo = f"select id from {schema}.rec_participant where equipment_number in "
# queryUnknownExternalEqNos = f"select equipment_number from {schema}.{TABLE_NAME} WHERE equipment_number not in "
# Ausstattungsnummern aller externer Teilnehmer, die getroffen wurden:
# query = f"select equipment_number from {schema}.{TABLE_NAME} \
# where equipment_number not in \
# (select equipment_number from {schema}.rec_participant where equipment_number > 0) \
# and position->>'reportingSource' = '3'"
# and originator_type = 4 AND position is null"
# query = f"select DISTINCT equipment_number from {schema}.{TABLE_NAME} \
# where equipment_number not in \
# (select equipment_number from {schema}.rec_participant where equipment_number > 0) ORDER BY equipment_number"
# query = f"select COUNT(*) FROM (SELECT DISTINCT equipment_number from {schema}.{TABLE_NAME} \
# where equipment_number not in \
# (select equipment_number from {schema}.rec_participant where equipment_number > 0) )"
print(f"▶️ {schema}")
try:
result = run_psql(query)
if result.strip():
print(result)
else:
print("(keine Ergebnisse)\n")
except subprocess.CalledProcessError as e:
print(f"❌ Fehler bei Schema {schema}: {e.stderr}\n")
if __name__ == "__main__":
main()
@@ -0,0 +1,228 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# $Id$
# $HeadURL$
# $Author$
#
from __future__ import print_function
from collections import defaultdict
import datetime
import subprocess
import sys
import os
class main_class(object):
def __init__( self ):
self.project = "guez"
self.database = "lts_ets_pro"
self.db_host = "localhost"
self.db_user = "postgres"
self.log_dir = '.'
self.log_file_name = "%s/update_db_%s.log" % (self.log_dir, self.project)
self.psql_command = 'psql'
self.pg_dump_command = 'pg_dump'
self.csv_file = 'guez_db_update_charge_types.csv'
self.chargeTypes = {}
def initialize(self):
self.log_hline()
self.log("# LOGFILE : %s" % os.path.abspath(self.log_file_name), True)
self.log("# DATE : %s" % datetime.datetime.today(), True)
self.log("# PROJECT : %s" % self.project, True)
self.log("# DATABASE : %s" % self.database, True)
self.log_hline()
def execute(self):
# execute sql files
self.log("# - Executings sql files on database %s ..." % self.database, True)
self.log_hline()
f = 'guez_db_update_charge_types_before.sql'
self.log("# - Executing sql file %s ..." % f, True)
self.execute_sql_file(f)
self.log("# => %s sql finished" % f, True)
self.log_hline()
self.log("# - Load charge types csv file %s" % self.csv_file, True)
self.chargeTypes = self.loadChargeTypes()
self.log_hline()
f = 'guez_db_create_charge_types_foreign_keys.sql'
exercises = ['ex_14', 'ex_17', 'ex_21', 'ex_22', 'ex_26', 'ex_27', 'ex_28', 'ex_29', 'ex_30', 'ex_31', 'ex_32', 'ex_33', 'ex_34', 'ex_35', 'ex_36', 'ex_37', 'ex_38', 'ex_39', 'ex_40', 'ex_41', 'ex_42', 'ex_44', 'ex_45', 'ex_46', 'ex_51']
tables = ['eem_event', 'rec_dynamic_participant_data', 'rec_history_state', 'rec_dus_mine_state', 'rec_equipment_gear_charge_state']
for ex in exercises:
self.dump_dynamic_tables(ex, tables)
self.log("# - Creating foreign keys (sql file %s) in schema %s" % (f, ex), True)
self.execute_sql_file_schema(f, ex)
self.log("# => %s finished for exercise %s " % (f, ex), True)
self.log_hline()
f = 'guez_db_update_charge_types_after.sql'
self.log("# - UPDATING ids - executing sql file %s ..." % f, True)
self.execute_sql_file(f)
self.log("# => %s sql finished" % f, True)
self.log_hline()
for ex in exercises:
for tabl in tables:
# replace charge types
old_file = ex+"_"+tabl+".sql"
new_file = ex+"_"+tabl+"_new.sql"
self.log("# - Replace charge types in file %s" % old_file, True)
db = self.read(old_file)
db = self.replaceChargeTypes(db)
self.write(new_file, db)
# delete from tables
self.log("# - Deleting dynamic data in schema %s" % ex, True)
cmd = '%s -h %s -U %s -c "DELETE FROM %s;" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, tabl, self.database, ex, self.log_file_name)
self.log(" Running command %s" % cmd, True)
ret = self.system(cmd)
if not ret:
self.die("=> Execution of delete commands %s failed\n => see logfile at %s\n\n" % (os.path.abspath(self.log_file_name)))
# insert new data
self.log("# - Inserting new dynamic data in schema %s" % ex, True)
self.execute_sql_file_schema(new_file, ex)
self.log("# => %s finished for exercise %s " % (new_file, ex), True)
self.log_hline()
self.log("# - Remove dynamic data files for exercise %s" % ex, True)
os.remove(old_file)
os.remove(new_file)
self.log_hline()
self.log("#### END ####", True)
self.log_hline()
return 0
def log_hline(self):
self.log("# -------------------------------------------------------------------------------", True)
def log(self, logline, log_to_stdout = False):
if self.log_file_name == None:
print("Logfile name not yet set - log: %s" % logline, file=sys.stderr)
return
with open( self.log_file_name, 'a' ) as f:
f.write(logline)
f.write("\n")
if log_to_stdout:
print(logline)
def die( self, message ):
self.log( message )
print("fatal error: %s" % message, file=sys.stderr)
sys.exit( 3 )
def system( self, cmdstring, show_output = False ):
self.log( "# Executing '%s'" % cmdstring, False)
if hasattr(sys, 'getwindowsversion'):
p = subprocess.Popen( cmdstring )
ret = p.wait()
else:
p = subprocess.Popen( cmdstring, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
ret = p.wait()
if show_output:
error = p.stdout.read()
print(error, file=sys.stderr)
if ( ret != 0 ):
print("Error executing %s" % cmdstring, file=sys.stderr)
return ret == 0
def execute_sql_file(self, f):
self.log( "Run sql-file %s " % f, True)
ret = self.system('%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=on %s >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, self.log_file_name))
if not ret:
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name)))
def execute_sql_file_schema(self, f, schema):
self.log( "Run sql-file %s " % f, True)
cmd = '%s -h %s -U %s -f "%s" --set ON_ERROR_STOP=ON "dbname=%s options=--search_path=%s" >> %s 2>&1' % (self.psql_command, self.db_host, self.db_user, f, self.database, schema, self.log_file_name)
self.log(" Running command %s" % cmd, True)
ret = self.system(cmd)
if not ret:
self.die("=> Execution of sql-commands for file %s failed\n => see logfile at %s\n\n" % (os.path.abspath(f), os.path.abspath(self.log_file_name)))
def dump_dynamic_tables(self, schema, tables):
self.log("# - Dump dynamic tables in schema %s" % schema, True)
# cmd = '%s -h %s -U %s -t %srec_dynamic_participant_data -t %seem_event -t %srec_dus_mine_state -t %srec_equipment_gear_charge_state -t %srec_history_state --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+".", schema+".", schema+".", schema+".", schema+".", self.database, schema+".sql")
for tabl in tables:
cmd = '%s -h %s -U %s -t %s --column-inserts --data-only %s > %s' % (self.pg_dump_command, self.db_host, self.db_user, schema+"."+tabl, self.database, schema+"_"+tabl+".sql")
self.log(" Running command %s" % cmd, True)
ret = self.system(cmd)
if not ret:
self.die("=> Execution of pg_dump failed\n => see logfile at %s\n\n" % os.path.abspath(self.log_file_name))
def read(self, filename):
"""Read a file and return a list of the lines."""
with open(filename, 'r') as f:
lines = f.read().split('\n')
return lines
def write(self, filename, lines):
"""Write a file for a given list of lines."""
with open(filename, 'w') as f:
for line in lines:
f.write(line + '\n')
def loadChargeTypes(self):
"""Read old and new charge types from a .csv file to a dictionary { old_id : new_id }."""
chargeTypes = {}
with open(self.csv_file, 'r') as f:
lines = f.read().split('\n')
for line in lines[1:]:
line = line.replace('\r', '')
try:
old, new = line.split(';')
if old in chargeTypes.keys():
print("duplicate old charge type &s".arg(old))
continue
chargeTypes[old] = new
except:
pass
return chargeTypes
def replaceChargeTypes(self,lines):
"""Replace all 'chargeTypeId' entries with new ones."""
def replace(iLine, line):
if "chargetypeid" in line.lower(): # ignore lines without charge type ID
items = line.split() # get all items in line separated by whitespace
for i, item in enumerate(map(lambda x: x.lower(), items)): # .lower() to catch all cases
if "chargetypeid" in item:
oldId = items[i+1].replace(',', '').replace('}', '').replace(']', '').replace("'", "") # remove characters to obtain ID
try:
newId = self.chargeTypes[oldId] # try lookup in chargeTypes
items[i+1] = items[i+1].replace(oldId, newId) # replace old ID with new ID
except:
if int(oldId) < -2: # any ID < -2 becomes 0
items[i+1] = items[i+1].replace(oldId, '0')
else: # -1 and -2 are valid IDs
pass
line = ' '.join(items) # rejoin items to new line
return line
return [replace(i, line) for i, line in enumerate(lines)] # simple list comprehension to get file content with updated IDs
def main():
p = main_class()
p.initialize()
p.execute()
sys.exit(0)
if __name__ == "__main__":
main()
@@ -0,0 +1,39 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 22 12:07:33 2021
@author: Filip Biedziak
Usage: guez_generate_js.py <csv file> <text file>
"""
#
# $Id$
# $HeadURL$
# $Author$
#
import sys
def toJson(assetBarcode, seBarcode):
string = ' exampleSocket.send(JSON.stringify({\n type: "mountTag",\n uuid: "UUID",\n data: {\n'
string += ' mountingPointTag: "' + assetBarcode + '",\n assetTag: "' + seBarcode + '"\n }\n }));\n'
return string
if __name__ == "__main__":
infile = sys.argv[1]
outfile = sys.argv[2]
strings = []
with open(infile, "r") as f:
lines = f.read().split("\n")
for line in lines[1:]:
if line == "":
continue
items = line.split(",")
if (len(items[1].replace('""','').strip()) == 0 or len(items[3].replace('""','').strip()) == 0):
continue
strings.append(toJson(items[1], items[3]))
with open(outfile, "w") as f:
f.write("function sendMountTags() {\n")
for s in strings:
f.write(s)
f.write("}")
@@ -0,0 +1,140 @@
#!/usr/bin/python
import csv
import os
import getopt
import sys
import time
class sql_generator(object):
def __init__( self ):
self.project = None
self.subdir = None
self.ins_ism_asset = list()
self.csv_header = None
self.default_header = []
def initialize(self):
if self.project == None:
print >> sys.stderr, "No project given"
sys.exit(1)
def execute(self):
print '\nDo not forget to convert the files from CP850 to UTF-8!\n'
print 'for f in `ls -1 *csv`; do iconv -f cp850 -t utf-8 -o $f.out $f; mv $f.out $f; done'
print 'or'
print 'for f in `ls -1 *csv`; do iconv -f iso8859-1 -t utf-8 -o $f.out $f; mv $f.out $f; done\n\n'
print 'depending on the used excel version / export settings'
self.read_csv('ism_asset.csv', self.ism_asset);
f = open(self.project + '_ism_values_asset.sql', 'wt')
head = """-- # $Id$
-- # $HeadURL$
-- # $Author$"""
f.write(head)
warning = '\n\n--- This file is automatically generated by ism_asset_csv_to_sql.py from csv (xls) files - do not modify here directly!'
f.write(warning)
f.write('\nBEGIN;\n-- ism_assets\n')
f.write('\n'.join(str(e) for e in self.ins_ism_asset))
f.write('\nCOMMIT;\n-- END\n')
def read_csv(self, filename, func):
print 'parsing %s' % filename
reader = csv.reader(open(os.path.join('./', filename), 'rt'), delimiter = ';')
self.parseHeader(reader)
for a in reader:
func(a)
def parseHeader(self, reader):
self.translation_header = []
self.default_header = []
self.csv_header = next(reader, None)
for i in range(0, len(self.csv_header)):
h = self.csv_header[i]
if '[' in h and h.endswith(']'):
lang = h[h.index('[') + 1:-1]
if lang == self.default_language:
self.default_header.append(i)
else:
self.translation_header.append(i)
self.translation_lang.append(lang)
def toBool(self, val):
if val == '':
return 'false'
if val == 'true' or val == '1':
return 'true'
if val == 'false' or val == '0':
return 'false'
if bool(val):
return 'true'
return 'false'
def toInt(self, val):
if val == '':
return 'NULL'
return int(val, 0)
def toFloat(self, val):
val = val.replace(',', '.')
if val == '':
return 0
return float(val)
def toString(self, val):
if val == '':
return 'NULL'
return "'" + val + "'"
def toJson(self, val):
return self.toString(val) # for now
def ism_asset(self, line):
if len(line[0]) == 0:
return
# Bezeichnung;TKZ;ArtikelNr.;SerienNr;Menge
article_no = self.toInt(line[2])
serial_no = self.toInt(line[3])
lastdigits = str(serial_no)[-4:]
name = str(article_no) + "_" + str(serial_no)
#lastdigits.strip("0")
optical_code = "{ \"opticalCode\" : " + lastdigits.lstrip("0") + "}"
props = self.toJson(optical_code)
barcode = "OPS_" + str(article_no) + "0" + str(serial_no) + "_" + str(article_no) + "0" + str(serial_no) + "_0" + str(serial_no)
last_update = int(time.time()) * 1000
self.ins_ism_asset.append("INSERT INTO ism.ism_asset (name, mounting_point_type_id, barcode, asset_state_id, properties) VALUES (%s, 6, %s, 0, %s);" % (self.toString(name), self.toString(barcode), props))
def usage():
print "Usage: %s <Options>" % sys.argv[0]
print "usage: $0 -p <project>"
print " -p (--project) : guez / ahtc "
def main():
p = sql_generator()
try:
opts, args = getopt.getopt(sys.argv[1:], "p:", ["project"])
except getopt.GetoptError, err:
print >> sys.stderr, err
usage()
sys.exit(2)
for o, a in opts:
if o in ("-p", "--project"):
p.project = a
print >> sys.stderr, "PROJECT: %s" % a
p.initialize()
p.execute()
sys.exit(0)
if __name__ == "__main__":
main()
@@ -0,0 +1,46 @@
#!/usr/bin/python
from test_scenario_definitions import *
import time
# Participant Ids
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
PPM1_participant_id = str(' 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 18, 19, 20, 21, 24, 25, 57, 58, 30, 31, 32, 52, 53, 54, 55').split(', ')
PPM1_type_id = str(' 4, 5, 6, 7, 8, 9, 10, 11, 12, 55, 78, 78, 57, 57, 27, 29, 54, 54, 77, 77, 78, 78, 78, 57, 57').split(', ')
PPM1_party_id = str(' 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 3, 3, 3, 4, 4, 4, 4').split(', ')
PPM1_equip_no = str('1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1009, 1010, 3801, 3802, 4401, 4402, 3803, 4403, 2501, 2502, 2001, 2002, 2003, 2011, 2012, 2013, 2014').split(', ')
def generate_participant_positions():
start_time = int(time.time()) * 1000
stmts = []
data_lat = 0.925926;
data_lon = 0.155215;
for i in range(0, 25):
data_lat += 0.000002
start_time += 100
stmt = 'INSERT INTO ex_42.rec_participant_position ( participant_id, participant_type_id, party_id, pos_lon, pos_lat, pos_z, sample_time ) VALUES ( %s, %s, 2, %s, %s, 0, %s )' % (PPM1_participant_id[i], PPM1_type_id[i], data_lon, data_lat, start_time )
stmt += ';'
stmts.append(stmt)
for i in range(0, 25):
data_lon += 0.000005
start_time += 100
stmt = 'INSERT INTO ex_42.rec_participant_position ( participant_id, participant_type_id, party_id, pos_lon, pos_lat, pos_z, sample_time ) VALUES ( %s, %s, 2, %s, %s, 0, %s )' % (PPM1_participant_id[i], PPM1_type_id[i], data_lon, data_lat, start_time )
stmt += ';'
stmts.append(stmt)
return stmts
if __name__ == "__main__":
print '\n'.join(generate_participant_positions())
@@ -0,0 +1,506 @@
#!/usr/bin/python
import csv
import os
import getopt
import sys
class sql_generator(object):
def __init__( self ):
self.project = None
self.subdir = None
self.ins_gear_type = list()
self.ins_charge_type = list()
self.upd_charge_type = list()
self.ins_gear_type_charge_type = list()
self.ins_igniter_type = list()
self.ins_igniter_function = list()
self.ins_charge_type_igniter_type_igniter_function = list()
self.ins_load_type = list()
self.ins_charge_type_load_type = list()
self.ins_building_charge_effect = list()
self.ins_can_component_string = list()
self.ins_bite_component_text = list()
self.ins_pdm_equipment_system_event_filter = list()
self.ins_pdm_event_filter_hgr_mine_search = list()
self.ins_radio_group = list()
self.ins_fire_purpose = list()
self.ins_target_type = list()
self.ins_target_subtype = list()
self.ins_target_type_target_subtype = list()
self.ins_nuclear_preset = list()
self.ins_vulnerability_kind = list()
self.ins_translation = list()
self.csv_header = None
self.translation_header = []
self.translation_lang = []
self.default_header = []
self.default_language = 'de_DE'
def initialize(self):
if self.project == None:
print >> sys.stderr, "No project given"
sys.exit(1)
def execute(self):
self.subdir = self.project+'_pdm'
print >> sys.stderr, "Looking into dir %s" % self.subdir
print '\nDo not forget to convert the files from CP850 to UTF-8!\n'
print 'for f in `ls -1 *csv`; do iconv -f cp850 -t utf-8 -o $f.out $f; mv $f.out $f; done'
print 'or'
print 'for f in `ls -1 *csv`; do iconv -f iso8859-1 -t utf-8 -o $f.out $f; mv $f.out $f; done\n\n'
print 'depending on the used excel version / export settings'
self.read_csv('charge_type.csv', self.charge_type);
self.read_csv('gear_type.csv', self.gear_type);
self.read_csv('igniter_type.csv', self.igniter_type);
self.read_csv('igniter_function.csv', self.igniter_function);
self.read_csv('charge_type_igniter_type_igniter_function.csv', self.charge_type_igniter_type_igniter_function);
self.read_csv('load_type.csv', self.load_type);
self.read_csv('charge_type_load_type.csv', self.charge_type_load_type);
self.read_csv('building_charge_effect.csv', self.building_charge_effect);
self.read_csv('can_component_string.csv', self.can_component_string);
self.read_csv('bite_component_text.csv', self.bite_component_text);
self.read_csv('event_filter_hgr_mine_search.csv', self.event_filter_hgr_mine_search);
self.read_csv('equipment_system_event_filter.csv', self.equipment_system_event_filter);
self.read_csv('radio_group.csv', self.radio_group);
self.read_csv('fire_purpose.csv', self.fire_purpose);
self.read_csv('target_type.csv', self.target_type);
self.read_csv('target_subtype.csv', self.target_subtype);
self.read_csv('target_type_target_subtype.csv', self.target_type_target_subtype);
self.read_csv('nuclear_preset.csv', self.nuclear_preset);
self.read_csv('vulnerability_kind.csv', self.vulnerability_kind);
# self.read_csv('type_class.csv', self.vulnerability_kind);
# self.read_csv('gear_class.csv', self.vulnerability_kind);
f = open(self.project + '_pdm_values.sql', 'wt')
head = """-- # $Id$
-- # $HeadURL$
-- # $Author$"""
f.write(head)
warning = '\n\n--- This file is automatically generated by pdm_csv_to_sql.py from csv (xls) files - do not modify here directly!'
f.write(warning)
f.write('\nBEGIN;\n-- pdm_gear_type\n')
f.write('\n'.join(str(e) for e in self.ins_gear_type))
f.write(warning)
f.write('\n\n-- charge_type\n')
f.write('\n'.join(str(e) for e in self.ins_charge_type))
f.write(warning)
f.write('\n\n-- gear_type_charge_type\n')
f.write('\n'.join(str(e) for e in self.ins_gear_type_charge_type))
f.write(warning)
f.write('\n\n-- igniter_type\n')
f.write('\n'.join(str(e) for e in self.ins_igniter_type))
f.write(warning)
f.write('\n\n-- igniter_function\n')
f.write('\n'.join(str(e) for e in self.ins_igniter_function))
f.write(warning)
f.write('\n\n-- charge_type_igniter_type_igniter_function\n')
f.write('\n'.join(str(e) for e in self.ins_charge_type_igniter_type_igniter_function))
f.write(warning)
f.write('\n\n-- load_type\n')
f.write('\n'.join(str(e) for e in self.ins_load_type))
f.write(warning)
f.write('\n\n-- charge_type_load_type\n')
f.write('\n'.join(str(e) for e in self.ins_charge_type_load_type))
f.write(warning)
f.write('\n\n-- building_charge_effect\n')
f.write('\n'.join(str(e) for e in self.ins_building_charge_effect))
f.write(warning)
f.write('\n\n-- can_component_string\n')
f.write('\n'.join(str(e) for e in self.ins_can_component_string))
f.write(warning)
f.write('\n\n-- bite_component_text\n')
f.write('\n'.join(str(e) for e in self.ins_bite_component_text))
f.write(warning)
f.write('\n\n-- pdm_equipment_system_event_filter\n')
f.write('\n'.join(str(e) for e in self.ins_pdm_equipment_system_event_filter))
f.write(warning)
f.write('\n\n-- pdm_event_filter_hgr_mine_search\n')
f.write('\n'.join(str(e) for e in self.ins_pdm_event_filter_hgr_mine_search))
f.write(warning)
f.write('\n\n-- radio_group\n')
f.write('\n'.join(str(e) for e in self.ins_radio_group))
f.write(warning)
f.write('\n\n-- fire_purpose\n')
f.write('\n'.join(str(e) for e in self.ins_fire_purpose))
f.write(warning)
f.write('\n\n-- target_type\n')
f.write('\n'.join(str(e) for e in self.ins_target_type))
f.write(warning)
f.write('\n\n-- target_subtype\n')
f.write('\n'.join(str(e) for e in self.ins_target_subtype))
f.write(warning)
f.write('\n\n-- targe_type_target_subtype\n')
f.write('\n'.join(str(e) for e in self.ins_target_type_target_subtype))
f.write(warning)
f.write('\n\n-- nuclear_preset\n')
f.write('\n'.join(str(e) for e in self.ins_nuclear_preset))
f.write(warning)
f.write('\n\n-- vulnerability_kind\n')
f.write('\n'.join(str(e) for e in self.ins_vulnerability_kind))
f.write(warning)
f.write('\n\n-- translation\n')
f.write('\n'.join(str(e) for e in self.ins_translation))
f.write('\nCOMMIT;\n-- END\n')
f2 = open(self.project + '_pdm_charge_type_update.sql', 'wt')
f2.write(head)
f2.write(warning)
f2.write('\nBEGIN;\n-- charge_type (update)\n')
f2.write('\n'.join(str(e) for e in self.upd_charge_type))
f2.write(warning)
f2.write('\nCOMMIT;\n-- END\n')
def read_csv(self, filename, func):
print 'parsing %s' % filename
reader = csv.reader(open(os.path.join(self.subdir, filename), 'rt'), delimiter = ';')
self.parseHeader(reader)
for a in reader:
func(a)
def parseHeader(self, reader):
self.translation_header = []
self.default_header = []
self.csv_header = next(reader, None)
for i in range(0, len(self.csv_header)):
h = self.csv_header[i]
if '[' in h and h.endswith(']'):
lang = h[h.index('[') + 1:-1]
if lang == self.default_language:
self.default_header.append(i)
else:
self.translation_header.append(i)
self.translation_lang.append(lang)
def toBool(self, val):
if val == '':
return 'false'
if val == 'true' or val == '1':
return 'true'
if val == 'false' or val == '0':
return 'false'
if bool(val):
return 'true'
return 'false'
def toInt(self, val):
if val == '':
return 'NULL'
return int(val, 0)
def toFloat(self, val):
val = val.replace(',', '.')
if val == '':
return 0
return float(val)
def toString(self, val):
if val == '':
return 'NULL'
return "'" + val + "'"
def toJson(self, val):
return self.toString(val) # for now
def create_translation(self, array_idx, line, context, disambiguation):
iso_code = self.translation_lang[array_idx]
key = line[self.default_header[array_idx]]
i18n = line[self.translation_header[array_idx]]
if len(i18n) == 0:
return
self.ins_translation.append("INSERT INTO pdm.pdm_translation (iso_code, disambiguation, context, key, translation) VALUES (%s, %s, %s, %s, %s);" % (self.toString(iso_code), self.toString(disambiguation), self.toString(context), self.toString(key), self.toString(i18n)))
def gear_type(self, line):
if len(line[0]) == 0:
return
# id;tas_code;name [de_DE];name [en_GB];gear_class_id
id = self.toInt(line[0])
self.ins_gear_type.append("INSERT INTO pdm.pdm_gear_type (id, tas_code, name, gear_class_id, properties) VALUES (%s, %s, %s, %s, %s);" % (id, self.toInt(line[1]), self.toString(line[2]), self.toInt(line[4]), self.toString(line[5])))
self.create_translation(0, line, 'GearType', 'name')
def charge_type(self, line):
if len(line[0]) == 0:
return
# 0- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14
# id,legacy_id,given_name,name [de_DE],shortName [de_DE],name [en_GB],shortName [en_GB],properties,gear_type,max_range,properties,gear_type,max_range,properties,dis_code
id = self.toInt(line[0])
legacy_id = self.toInt(line[1])
predefined_name = self.toString(line[2])
name = self.toString(line[3])
short_name = self.toString(line[4])
props = self.toJson(line[7])
dis_entity = self.toJson(line[14]) if len(line) > 14 else 'NULL';
properties_1 = self.toJson(line[10])
properties_2 = self.toJson(line[13])
self.ins_charge_type.append("INSERT INTO pdm.pdm_charge_type (id, predefined_name, name, short_name, properties, dis_entity, charge_legacy_id) VALUES (%s, %s, %s, %s, %s, %s, %s);" % (id, predefined_name, name, short_name, props, dis_entity, legacy_id))
self.upd_charge_type.append("UPDATE pdm.pdm_charge_type SET predefined_name = %s, name = %s, short_name = %s, properties = %s, dis_entity = %s, charge_legacy_id = %s WHERE ID = %s;" % (predefined_name, name, short_name, props, dis_entity, legacy_id, id))
for i in (8,11):
if i >= len(line): break
gt = line[i]
if gt == '': continue
range = ''
if i + 1 < len(line):
range = line[i+1]
if properties_1 == "NULL":
self.ins_gear_type_charge_type.append("INSERT INTO pdm.pdm_gear_type_charge_type (gear_type_id, charge_type_id, max_range) VALUES (%s, %s, %s);" % (gt, id, self.toInt(range)))
else:
self.ins_gear_type_charge_type.append("INSERT INTO pdm.pdm_gear_type_charge_type (gear_type_id, charge_type_id, max_range, properties) VALUES (%s, %s, %s, %s);" % (gt, id, self.toInt(range), properties_1))
self.create_translation(0, line, 'ChargeType', 'name')
self.create_translation(1, line, 'ChargeType', 'shortName')
def igniter_type(self, line):
if len(line[0]) == 0:
return
# id;name [de_DE];shortName [de_DE],name [en_GB];shortName [en_GB];properties
id = self.toInt(line[0])
name = self.toString(line[1])
short_name = self.toString(line[2])
props = self.toJson(line[5])
self.ins_igniter_type.append("INSERT INTO pdm.pdm_igniter_type (id, name, short_name, properties) VALUES (%s, %s, %s, %s);" % (id, name, short_name, props))
self.create_translation(0, line, 'IngiterType', 'name')
self.create_translation(1, line, 'IgniterType', 'shortName')
def igniter_function(self, line):
if len(line[0]) == 0:
return
# id;name [de_DE];shortName [de_DE],name [en_GB];shortName [en_GB];properties
id = self.toInt(line[0])
name = self.toString(line[1])
short_name = self.toString(line[2])
props = self.toJson(line[5])
self.ins_igniter_function.append("INSERT INTO pdm.pdm_igniter_function (id, name, short_name, properties) VALUES (%s, %s, %s, %s);" % (id, name, short_name, props))
self.create_translation(0, line, 'IngiterFunction', 'name')
self.create_translation(1, line, 'IgniterFunction', 'shortName')
def charge_type_igniter_type_igniter_function(self, line):
if len(line[0]) == 0:
return
# charge_type_id;igniter_type_id;igniter_function_id;impact_area,impact_radius;hazard_distance;charge_legacy_id;igniter_function_base_id;properties
charge_type_id = self.toInt(line[0])
igniter_type_id = self.toInt(line[1])
igniter_function_id = self.toInt(line[2])
impact_area = self.toJson(line[3])
impact_radius = self.toInt(line[4])
hazard_distance = self.toInt(line[5])
charge_legacy_id = self.toInt(line[6])
igniter_function_base_id = self.toInt(line[7])
props = self.toJson(line[8])
self.ins_charge_type_igniter_type_igniter_function.append("INSERT INTO pdm.pdm_charge_type_igniter_type_igniter_function (charge_type_id, igniter_type_id, igniter_function_id, impact_area, impact_radius, hazard_distance, charge_legacy_id, igniter_function_base_id, properties) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s);" % (charge_type_id, igniter_type_id, igniter_function_id, impact_area, impact_radius, hazard_distance, charge_legacy_id, igniter_function_base_id, props))
def load_type(self, line):
if len(line[0]) == 0:
return
# id;name [de_DE];shortName [de_DE],name [en_GB];shortName [en_GB];properties
id = self.toInt(line[0])
name = self.toString(line[1])
short_name = self.toString(line[2])
props = self.toJson(line[5])
self.ins_load_type.append("INSERT INTO pdm.pdm_load_type (id, name, short_name, properties) VALUES (%s, %s, %s, %s);" % (id, name, short_name, props))
self.create_translation(0, line, 'LoadType', 'name')
self.create_translation(1, line, 'LoadType', 'shortName')
def charge_type_load_type(self, line):
if len(line[0]) == 0:
return
# charge_type_id;load_type_id;numbers;properties
charge_type_id = self.toInt(line[0])
load_type_id = self.toInt(line[1])
numbers = self.toString(line[2])
props = self.toJson(line[3])
self.ins_charge_type_load_type.append("INSERT INTO pdm.pdm_charge_type_load_type (charge_type_id, load_type_id, numbers, properties) VALUES (%s, %s, %s, %s);" % (charge_type_id, load_type_id, numbers, props))
def building_charge_effect(self, line):
if len(line[0]) == 0:
return
# id, charge_type_id;destructive_factor;penetration_factor;hit_sound;difcue_light;difcue_pyro
ky = self.toInt(line[0])
ct = self.toInt(line[1])
df = self.toInt(line[2])
pf = self.toInt(line[3])
hs = self.toInt(line[4])
dl = self.toBool(line[5])
dp = self.toBool(line[6])
self.ins_building_charge_effect.append("INSERT INTO pdm.pdm_building_charge_effect (id, charge_type_id, destructive_factor, penetration_factor, hit_sound, difcue_light, difcue_pyro) VALUES (%s, %s, %s, %s, %s, %s, %s);" % (ky, ct, df, pf, hs, dl, dp))
def can_component_string(self, line):
if len(line[0]) == 0:
return
# name [de_DE], name [en_GB], bit_mask, can_device_id, sub_device_id
name = self.toString(line[0])
bm = self.toInt(line[2])
cd = self.toInt(line[3])
sd = self.toInt(line[4])
self.ins_can_component_string.append("INSERT INTO pdm.pdm_can_component_string (name, bit_mask, can_device_id, sub_device_id) VALUES (%s, %s, %s, %s);" % (name, bm, cd, sd))
self.create_translation(0, line, 'CanComponentString', 'name')
def bite_component_text(self, line):
if len(line[0]) == 0:
return
# can_device_id, sub_device_id, error_number, name
name = self.toString(line[0])
cd = self.toInt(line[2])
sd = self.toInt(line[3])
er = self.toInt(line[4])
self.ins_bite_component_text.append("INSERT INTO pdm.pdm_bite_component_text (name, can_device_id, sub_device_id, error_number) VALUES (%s, %s, %s, %s);" % (name, cd, sd, er))
self.create_translation(0, line, 'BiteComponentText', 'name')
def equipment_system_event_filter(self, line):
if len(line[0]) == 0:
return
# equipment_functional_id, sub_device_id, device_specific_code, hgr_mine_search_filter_type_id
ef = self.toInt(line[0])
sd = self.toInt(line[1])
dsc = self.toInt(line[2])
hgr = self.toInt(line[3])
self.ins_pdm_equipment_system_event_filter.append("INSERT INTO pdm.pdm_equipment_system_event_filter (equipment_functional_id, sub_device_id, device_specific_code, hgr_mine_search_filter_type_id) VALUES (%s, %s, %s, %s);" % (ef, sd, dsc, hgr))
def event_filter_hgr_mine_search(self, line):
if len(line[0]) == 0:
return
# hgr_mine_search_filter_type_idm, charge_type_id
hgr = self.toInt(line[0])
ct = self.toInt(line[1])
self.ins_pdm_event_filter_hgr_mine_search.append("INSERT INTO pdm.pdm_event_filter_hgr_mine_search (hgr_mine_search_filter_type_id, charge_type_id) VALUES (%s, %s);" % (hgr, ct))
def radio_group(self, line):
if len(line[0]) == 0:
return
txt = (self.toInt(line[0]), self.toString(line[1]), self.toBool(line[2]), self.toInt(line[3]), self.toInt(line[4]), self.toInt(line[5]), self.toString(line[6]), self.toInt(line[7]), self.toString(line[8]), self.toInt(line[9]))
if len(line) > 10:
self.create_translation(0, line, 'RadioGroup', 'name')
# id, name[def], base_group, issi, group_issi1, group_issi2, radio_hostname, radio_port, radio_backup_hostname, radio_backup_port, name[en_GB] (optional)
self.ins_radio_group.append("INSERT INTO pdm.pdm_radio_group (id, name, base_group, issi, group_issi1, group_issi2, radio_hostname, radio_port, radio_backup_hostname, radio_backup_port) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);" % txt);
def fire_purpose(self, line):
if len(line[0]) == 0:
return
# id, name [de_DE], name [en_GB], Description, properties
id = self.toInt(line[0])
name = self.toString(line[1])
props = self.toJson(line[4])
txt = (id, name, props)
self.ins_fire_purpose.append("INSERT INTO pdm.pdm_fire_purpose (id, name, properties) VALUES (%s, %s, %s);" % txt)
self.create_translation(0, line, 'FirePurpose', 'name')
def target_type(self, line):
if len(line[0]) == 0:
return
# id, name [de_DE], name [en_GB], abbreviation
id = self.toInt(line[0])
name = self.toString(line[1])
abbr = self.toString(line[3])
props = self.toJson(line[4])
txt = (id, name, abbr, props)
self.ins_target_type.append("INSERT INTO pdm.pdm_target_type (id, name, abbreviation, properties) VALUES (%s, %s, %s, %s);" % txt)
self.create_translation(0, line, 'TargetType', 'name')
def target_subtype(self, line):
if len(line[0]) == 0:
return
# id, name [de_DE], name [en_GB], abbreviation
id = self.toInt(line[0])
name = self.toString(line[1])
abbr = self.toString(line[3])
props = self.toJson(line[4])
txt = (id, name, abbr, props)
self.ins_target_subtype.append("INSERT INTO pdm.pdm_target_subtype (id, name, abbreviation, properties) VALUES (%s, %s, %s, %s);" % txt)
self.create_translation(0, line, 'TargetSubType', 'name')
def target_type_target_subtype(self, line):
if len(line[0]) == 0:
return
# target_type_id;target_subtype_id
targetTypeId = self.toInt(line[0])
targetSubtypeId = self.toString(line[1])
props = self.toJson(line[2])
txt = (targetTypeId, targetSubtypeId, props)
self.ins_target_type_target_subtype.append("INSERT INTO pdm.pdm_target_type_target_subtype (target_type_id, target_subtype_id, properties) VALUES (%s, %s, %s);" % txt)
def nuclear_preset(self, line):
if len(line[0]) == 0:
return
# id name [de_DE] name [en_GB] detonation_value impact_radius border_length_inner border_length_outer
id = self.toInt(line[0])
name = self.toString(line[1])
det = self.toFloat(line[3])
imp = self.toInt(line[4])
bli = self.toInt(line[5])
blo = self.toInt(line[6])
txt = (id, name, det, imp, bli, blo)
self.ins_nuclear_preset.append("INSERT INTO pdm.pdm_nuclear_preset (id, name, detonation_value, impact_radius, border_length_inner, border_length_outer) VALUES (%s, %s, %s, %s, %s, %s);" % txt)
self.create_translation(0, line, 'NuclearPreset', 'name')
def vulnerability_kind(self, line):
if len(line[0]) == 0:
return
# player_type_class_id;ta_vulnerability_kind;vulnerability_class_id;name [de_DE];name [en_GB];Teilnehmer-Name;Teilnehmer-Typ;Index;Fallback
id = self.toInt(line[0])
kind = self.toInt(line[1])
cls = self.toInt(line[2])
name = self.toString(line[3])
txt = (id, kind, cls, name)
self.ins_vulnerability_kind.append("INSERT INTO pdm.pdm_vulnerability_kind (player_type_class_id, ta_vulnerability_kind, vulnerability_class_id, name) VALUES (%s, %s, %s, %s);" % txt)
self.create_translation(0, line, 'VulnerabilityKind', 'name')
def usage():
print "Usage: %s <Options>" % sys.argv[0]
print "usage: $0 -p <project>"
print " -p (--project) : guez / ahtc "
def main():
p = sql_generator()
try:
opts, args = getopt.getopt(sys.argv[1:], "p:", ["project"])
except getopt.GetoptError, err:
print >> sys.stderr, err
usage()
sys.exit(2)
for o, a in opts:
if o in ("-p", "--project"):
p.project = a
print >> sys.stderr, "PROJECT: %s" % a
p.initialize()
p.execute()
sys.exit(0)
if __name__ == "__main__":
main()
@@ -0,0 +1,10 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Participant Ids
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
PPM1_participant_id = str(' 5, 6, 7, 8, 9, 12, 13, 14, 15, 16, 18, 19, 20, 21, 24, 25, 57, 58, 30, 31, 32, 52, 53, 54, 55, 90, 91, 92, 93, 95, 96, 97, 98').split(', ')
PPM1_type_id = str(' 55, 55, 55, 74, 74, 55, 55, 55, 55, 55, 78, 78, 57, 57, 27, 29, 54, 54, 77, 77, 78, 78, 78, 57, 57, 173, 173, 173, 173, 173, 173, 173, 173').split(', ')
PPM1_party_id = str(' 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 3, 3, 3, 4, 4, 4, 4, 2, 2, 2, 2, 3, 3, 3, 3').split(', ')
PPM1_equip_no = str('1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 3801, 3802, 4401, 4402, 3803, 4403, 2501, 2502, 2001, 2002, 2003, 2011, 2012, 2013, 2014, 0, 0, 0, 0, 0, 0, 0, 0').split(', ')
@@ -0,0 +1,47 @@
#!/usr/bin/python
from test_scenario_definitions import *
import time
def generate_participant_positions():
start_time = int(time.time()) * 1000
stmts = []
data_lat = 0.925926;
data_lon = 0.155215;
for i in range(0, 25):
data_lat += 0.000002
start_time += 100
stmt = 'INSERT INTO ex_42.rec_participant_position ( participant_id, participant_type_id, party_id, pos_lon, pos_lat, pos_z, sample_time ) VALUES ( %s, %s, 2, %s, %s, 0, %s )' % (PPM1_participant_id[i], PPM1_type_id[i], data_lon, data_lat, start_time )
stmt += ';'
stmts.append(stmt)
for i in range(0, 25):
data_lon += 0.000005
start_time += 100
stmt = 'INSERT INTO ex_42.rec_participant_position ( participant_id, participant_type_id, party_id, pos_lon, pos_lat, pos_z, sample_time ) VALUES ( %s, %s, 2, %s, %s, 0, %s )' % (PPM1_participant_id[i], PPM1_type_id[i], data_lon, data_lat, start_time )
stmt += ';'
stmts.append(stmt)
data_lat = 0.929926;
data_lon = 0.156215;
for i in range(26, 33):
data_lat += 0.000002
start_time += 100
stmt = 'INSERT INTO ex_42.rec_participant_position ( participant_id, participant_type_id, party_id, pos_lon, pos_lat, pos_z, sample_time ) VALUES ( %s, %s, 2, %s, %s, 0, %s )' % (PPM1_participant_id[i], PPM1_type_id[i], data_lon, data_lat, start_time )
stmt += ';'
stmts.append(stmt)
return stmts
if __name__ == "__main__":
print '\n'.join(generate_participant_positions())
@@ -0,0 +1,179 @@
#!/usr/bin/python
from test_scenario_definitions import *
import time
import json
def generate_participant_state():
start_time = int(time.time()) * 1000
stmts = []
# sanitation material
#start_time += 100
stock = []
for i in range(1, 5):
tmp1 = dict()
tmp1['materialId'] = i;
tmp1['materialCount'] = i*10;
stock.append(tmp1)
json_str_stock = json.dumps(stock)
#stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data (sanitation_material_stock, participant_id, sample_time, sanitation_unique_id, qualification ) VALUES (\'%s\', %s, %s, %s, %s )' % (json_str, PPM1_participant_id[i], start_time, 123, 0 )
#stmt += ';'
#stmts.append(stmt)
# firings
for i in range(0, 9):
start_time += 100
tmp = dict()
if ((i%2) == 0):
tmp['originatorChargeTypeId'] = 38;
else:
tmp['originatorChargeTypeId'] = 39;
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = i;
tmp['originatorId'] = 0;
tmp['originatorType'] = 1; #participant
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 2**i;
data = []
data.append(tmp)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, sanitation_material_stock, sanitation_unique_id, qualification, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', \'%s\', \'%s\', \'%s\', false, false, %s, %s )' % (json_str, json_str_stock, 123, 2, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
# nbc
for i in range(13, 15):
start_time += 100
tmp = dict()
tmp['originatorChargeTypeId'] = 341; #chemical agent
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = 5;
tmp['originatorId'] = 3; #nbc
tmp['originatorType'] = 2; #area effect simulation
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 2**5;
data = dict()
data['ParticipantManagement::StateInformation'] = []
data['ParticipantManagement::StateInformation'].append(tmp)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', false, false, %s, %s )' % (json_str, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
# mine area
for i in range(15, 17):
start_time += 100
tmp = dict()
tmp['originatorChargeTypeId'] = 351; #mine area
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = 8;
tmp['originatorId'] = 2; #mine
tmp['originatorType'] = 2; #area effect simulation
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 2**8;
data = dict()
data['ParticipantManagement::StateInformation'] = []
data['ParticipantManagement::StateInformation'].append(tmp)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', false, false, %s, %s )' % (json_str, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
# functional area
for i in range(10, 13):
start_time += 100
tmp = dict()
tmp['originatorChargeTypeId'] = 411; #functional area
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = 0;
tmp['originatorId'] = 4; #functional area
tmp['originatorType'] = 2; #area effect simulation
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 0;
data = dict()
data['ParticipantManagement::StateInformation'] = []
data['ParticipantManagement::StateInformation'].append(tmp)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', false, false, %s, %s )' % (json_str, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
# explosive device
for i in range(18, 21):
start_time += 100
tmp = dict()
if ((i%2) == 0):
tmp['originatorChargeTypeId'] = 48;
else:
tmp['originatorChargeTypeId'] = 49;
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = i-17; #1,2,3
tmp['originatorId'] = 1; #ied 1
tmp['originatorType'] = 20; #improvisedExplosiveDevice
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 2**(i-17);
data = []
data.append(tmp)
# tmp1 = dict()
# tmp1['originatorChargeTypeId'] = 70;
# tmp1['stateTime'] = start_time;
# tmp1['vulnerabilityState'] = 8;
# tmp1['originatorId'] = 1; #ied 1
# tmp1['originatorType'] = 20; #improvisedExplosiveDevice
# tmp1['vulnerabilityKindId'] = 289;
# tmp1['vulnerabilityBits'] = 256;
# data.append(tmp1)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', false, false, %s, %s )' % (json_str, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
# air support
for i in range(21, 24):
start_time += 100
tmp = dict()
tmp['originatorChargeTypeId'] = 471;
tmp['stateTime'] = start_time;
tmp['vulnerabilityState'] = 8;
tmp['originatorId'] = 0;
tmp['originatorType'] = 2; #area effect simulation
tmp['vulnerabilityKindId'] = 0;
tmp['vulnerabilityBits'] = 2**8;
data = []
data.append(tmp)
json_str = json.dumps(data)
stmt = 'INSERT INTO ex_42.rec_dynamic_participant_data ( state_history, has_nbc_protection_active, has_nbc_protection_configured, participant_id, sample_time ) VALUES (\'%s\', false, false, %s, %s )' % (json_str, PPM1_participant_id[i], start_time )
stmt += ';'
stmts.append(stmt)
return stmts
if __name__ == "__main__":
print '\n'.join(generate_participant_state())
@@ -0,0 +1,48 @@
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import sys
from collections import OrderedDict
#asset_id,asset_barcode,structure_element_id,se_barcode,main_structure_element_id,name
def readBarcodes(filename):
"""Create a dictionary { name, MSE-name : barcode }."""
ret = OrderedDict()
with open(filename, "r") as f:
lines = f.read().split("\n")
cols = lines[0].strip().split(',')
#assert cols[0].lower() == "asset_id"
assert cols[1].lower() == "asset_barcode"
assert cols[3].lower() == "se_barcode"
rows = [line.strip() for line in lines[1:] if line.strip() != ""]
for row in rows:
items = row.split(",")
ret[items[1]] = items[3]
return ret
def appendBarcodeUpdates(outputFile, barcodes):
"""Append an SQL INSERT statement for barcodes."""
lines = []
for key in barcodes.keys():
asset_barcode = key
update = "INSERT INTO ism.ism_mounting_point_asset (asset_id, structure_element_id) VALUES ((SELECT id FROM ism.ism_asset WHERE barcode = '%s'), (SELECT id FROM ism.ism_structure_element WHERE barcode = '%s' ));" % (asset_barcode, barcodes[key])
lines.append(update)
with open(outputFile, "w") as f:
for line in lines:
f.write(line + "\n")
if __name__ == "__main__":
barcodeFile = sys.argv[1]
barcodes = readBarcodes(barcodeFile)
sqlOutputFile = sys.argv[2]
appendBarcodeUpdates(sqlOutputFile, barcodes)
@@ -0,0 +1,47 @@
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import sys
from collections import OrderedDict
def readBarcodes(filename):
"""Create a dictionary { name, MSE-name : barcode }."""
ret = OrderedDict()
with open(filename, "r") as f:
lines = f.read().split("\n")
cols = lines[0].strip().split(',')
assert cols[1].lower() == "name"
#assert cols[2].lower() == "main_structure_element_id"
assert cols[3].lower() == "main_name"
assert cols[4].lower() == "barcode"
rows = [line.strip() for line in lines[1:] if line.strip() != ""]
for row in rows:
items = row.split(",")
ret[items[1], items[3]] = items[4]
return ret
def appendBarcodeUpdates(outputFile, barcodes):
"""Append an SQL UPDATE statement for barcodes."""
lines = []
for key in barcodes.keys():
name, main_name = key
update = "UPDATE ism.ism_structure_element SET barcode = '%s' WHERE main_structure_element_id = (SELECT id FROM ism.ism_main_structure_element WHERE name = '%s') AND name = '%s';" % (barcodes[key], main_name, name)
lines.append(update)
with open(outputFile, "w") as f:
for line in lines:
f.write(line + "\n")
if __name__ == "__main__":
barcodeFile = sys.argv[1]
barcodes = readBarcodes(barcodeFile)
sqlOutputFile = sys.argv[2]
appendBarcodeUpdates(sqlOutputFile, barcodes)
@@ -0,0 +1,67 @@
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import sys
def read(filename):
"""Read a file and return a list of the lines."""
with open(filename, 'r') as f:
lines = f.read().split('\n')
return lines
def write(filename, lines):
"""Write a file for a given list of lines."""
with open(filename, 'w') as f:
for line in lines:
f.write(line + '\n')
def loadChargeTypes(filename):
"""Read old and new charge types from a .csv file to a dictionary { old_id : new_id }."""
chargeTypes = {}
with open(filename, 'r') as f:
lines = f.read().split('\n')
for line in lines[1:]:
line = line.replace('\r', '')
try:
old, new = line.split(';')
if old in chargeTypes.keys():
print("duplicate old charge type &s".arg(old))
continue
chargeTypes[old] = new
except:
pass
return chargeTypes
def replaceChargeTypes(lines, chargeTypes):
"""Replace all 'chargeTypeId' entries with new ones."""
def replace(iLine, line):
if "chargetypeid" in line.lower(): # ignore lines without charge type ID
items = line.split() # get all items in line separated by whitespace
for i, item in enumerate(map(lambda x: x.lower(), items)): # .lower() to catch all cases
if "chargetypeid" in item:
oldId = items[i+1].replace(',', '').replace('}', '').replace(']', '').replace("'",'') # remove characters to obtain ID
try:
newId = chargeTypes[oldId] # try lookup in chargeTypes
items[i+1] = items[i+1].replace(oldId, newId) # replace old ID with new ID
except:
if int(oldId) < -2: # any ID < -2 becomes 0
items[i+1] = items[i+1].replace(oldId, '0')
else: # -1 and -2 are valid IDs
pass
line = ' '.join(items) # rejoin items to new line
return line
return [replace(i, line) for i, line in enumerate(lines)] # simple list comprehension to get file content with updated IDs
if __name__ == "__main__":
chargeTypes = loadChargeTypes(sys.argv[1])
db = read(sys.argv[2])
db = replaceChargeTypes(db, chargeTypes)
write(sys.argv[3], db)
@@ -0,0 +1,48 @@
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import sys
from collections import OrderedDict
#id,mounting_point_schedule_id,structure_element_id,main_structure_element_id,name,main_name
def readBarcodes(filename):
ret = OrderedDict()
with open(filename, "r") as f:
lines = f.read().split("\n")
cols = lines[0].strip().split(',')
assert cols[1].lower() == "mounting_point_schedule_id"
assert cols[2].lower() == "structure_element_id"
assert cols[3].lower() == "main_structure_element_id"
assert cols[4].lower() == "name"
assert cols[5].lower() == "main_name"
rows = [line.strip() for line in lines[1:] if line.strip() != ""]
for row in rows:
items = row.split(",")
ret[items[1], items[4], items[5]] = items[2]
return ret
def appendBarcodeUpdates(outputFile, barcodes):
"""Append an SQL INSERT statement for MPP."""
lines = []
for key in barcodes.keys():
mpp_id, name, main_name = key
update = "INSERT INTO ism.ism_mounting_point_plan (mounting_point_schedule_id, structure_element_id) VALUES (%s, (select se.id from ism.ism_structure_element se, ism.ism_main_structure_element mse where se.name = '%s' AND mse.id = se.main_structure_element_id AND mse.name = '%s'));" % (mpp_id, name, main_name)
lines.append(update)
with open(outputFile, "w") as f:
for line in lines:
f.write(line + "\n")
if __name__ == "__main__":
barcodeFile = sys.argv[1]
barcodes = readBarcodes(barcodeFile)
sqlOutputFile = sys.argv[2]
appendBarcodeUpdates(sqlOutputFile, barcodes)
+110 -48
View File
@@ -215,38 +215,7 @@ cmd_init_db() {
log_info "Initializing database for project: ${project}"
# Determine create_excon_db.py arguments
local db_args=""
case "${project}" in
guez)
db_args="-p guez"
;;
ahtc)
db_args="-p ahtc"
;;
qatar)
db_args="-p qatar"
;;
l400p2)
db_args="-p l400p2"
;;
demo)
db_args="-p demo"
;;
commaster)
db_args="-p commaster"
;;
test-db)
db_args="-p guez -e"
;;
*)
log_error "Unknown project: ${project}"
log_info "Valid projects: guez, ahtc, qatar, l400p2, demo, commaster, test-db"
return 1
;;
esac
# Find create_excon_db.py script
# Install Python3 in container temporarily and execute create_excon_db.py
local script_path="${SCRIPT_DIR}/Lts/Tests/DatabaseInitialisation/create_excon_db.py"
if [ ! -f "${script_path}" ]; then
@@ -254,28 +223,121 @@ cmd_init_db() {
return 1
fi
# Copy script to container and execute
log_info "Installing Python3 in container (temporary)..."
docker exec "${CONTAINER_NAME}" apt-get update -qq >/dev/null 2>&1 || true
docker exec "${CONTAINER_NAME}" apt-get install -y -qq python3 >/dev/null 2>&1 || {
log_warn "Could not install python3, using direct SQL execution instead"
# Fallback: execute SQL files directly
cmd_init_db_sql "${project}"
return $?
}
# Determine create_excon_db.py arguments
local db_args=""
case "${project}" in
guez) db_args="-p guez" ;;
ahtc) db_args="-p ahtc" ;;
qatar) db_args="-p qatar" ;;
l400p2) db_args="-p l400p2" ;;
demo) db_args="-p demo" ;;
commaster) db_args="-p commaster" ;;
test-db) db_args="-p guez -e" ;;
*)
log_error "Unknown project: ${project}"
return 1
;;
esac
log_info "Executing create_excon_db.py ${db_args}..."
docker exec -i "${CONTAINER_NAME}" /bin/bash -c "cat > /tmp/create_excon_db.py && chmod +x /tmp/create_excon_db.py" < "${script_path}"
# Copy script to container
docker cp "${script_path}" "${CONTAINER_NAME}:/tmp/create_excon_db.py"
docker exec "${CONTAINER_NAME}" chmod +x /tmp/create_excon_db.py
# Read lts-cc.conf for database configuration
# Copy lts-cc.conf if exists
local config_arg=""
if [ -f "${LTS_CC_CONF}" ]; then
local db_host db_name
db_host=$(grep -E "^DatabaseHostName=" "${LTS_CC_CONF}" | cut -d'=' -f2)
db_name=$(grep -E "^DatabaseName=" "${LTS_CC_CONF}" | cut -d'=' -f2)
# Execute with config file
docker exec -i "${CONTAINER_NAME}" /bin/bash -c "
cat > /tmp/lts-cc.conf
cat > /tmp/create_excon_db.py && chmod +x /tmp/create_excon_db.py
cd /tmp && python3 /tmp/create_excon_db.py ${db_args} -c /tmp/lts-cc.conf -b /tmp
" < "${LTS_CC_CONF}"
else
# Execute without config file (uses defaults)
docker exec "${CONTAINER_NAME}" python3 /tmp/create_excon_db.py ${db_args}
docker cp "${LTS_CC_CONF}" "${CONTAINER_NAME}:/tmp/lts-cc.conf"
config_arg="-c /tmp/lts-cc.conf"
fi
# Copy SQL files to container
local sql_dir="${SCRIPT_DIR}/Lts/Tests/DatabaseInitialisation"
docker exec "${CONTAINER_NAME}" mkdir -p /tmp/sql
for f in "${sql_dir}"/*.sql; do
if [ -f "$f" ]; then
docker cp "$f" "${CONTAINER_NAME}:/tmp/sql/"
fi
done
# Execute create_excon_db.py in container
docker exec -w /tmp "${CONTAINER_NAME}" python3 /tmp/create_excon_db.py ${db_args} ${config_arg} -b /tmp
log_info "Database initialization completed for project: ${project}"
}
# Fallback: Initialize database by executing SQL files directly
cmd_init_db_sql() {
local project="${1:-}"
local sql_dir="${SCRIPT_DIR}/Lts/Tests/DatabaseInitialisation"
log_info "Executing SQL files directly for project: ${project}"
# List of base SQL files to execute
local base_files=(
"init_db_users_grants.sql"
"init_db_procedures.sql"
"init_db_schemas.sql"
)
# Project-specific files
local project_files=()
case "${project}" in
guez)
project_files=(
"guez_ppm_values_types.sql"
"guez_ppm_values_templates.sql"
"guez_pdm_values.sql"
"guez_pdm_values_translation.sql"
"guez_opm_values_initial.sql"
"guez_opm_values.sql"
"guez_flm_symbols.sql"
"guez_update_flm_used_by.sql"
)
;;
ahtc)
project_files=(
"ahtc_ppm_values_types.sql"
"ahtc_ppm_values_templates.sql"
"ahtc_pdm_values.sql"
"ahtc_opm_values.sql"
"ahtc_flm_symbols.sql"
)
;;
*)
log_warn "Direct SQL execution not fully implemented for ${project}"
return 1
;;
esac
# Execute base files
for f in "${base_files[@]}"; do
if [ -f "${sql_dir}/${f}" ]; then
log_info "Executing: ${f}"
docker exec -i "${CONTAINER_NAME}" psql -U postgres -d lts_ets_pro -f "/tmp/sql/${f}" 2>&1 | head -5
fi
done
# Execute project files
for f in "${project_files[@]}"; do
if [ -f "${sql_dir}/${f}" ]; then
log_info "Executing: ${f}"
docker exec -i "${CONTAINER_NAME}" psql -U postgres -d lts_ets_pro -f "/tmp/sql/${f}" 2>&1 | head -5
else
log_warn "SQL file not found: ${f}"
fi
done
log_info "Database initialization completed for project: ${project}"
}
@@ -0,0 +1,74 @@
# DEB Packaging Build Script for Legatus Qt6
#
# This script builds all DEB packages from the packaging directory structure.
#
# Usage:
# ./build-deb-packages.sh [version]
#
# Example:
# ./build-deb-packages.sh 15.6-qt6.10.2
#
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
VERSION="${1:-15.6-qt6.10.2}"
OUTPUT_DIR="${SCRIPT_DIR}/output"
PACKAGING_DIR="${SCRIPT_DIR}/packaging/deb"
# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'
log_info() {
echo -e "${GREEN}[INFO]${NC} $1"
}
log_warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
log_error() {
echo -e "${RED}[ERROR]${NC} $1"
}
# Create output directory
mkdir -p "${OUTPUT_DIR}"
# List of packages to build
PACKAGES=("legatus-qt6-sv" "legatus-qt6-ws" "legatus-qt6-aar" "legatus-qt6-ltscc" "legatus-qt6-licence")
# Build each package
for pkg in "${PACKAGES[@]}"; do
log_info "Building package: ${pkg}_${VERSION}_amd64.deb"
pkg_dir="${PACKAGING_DIR}/${pkg}"
if [ ! -d "${pkg_dir}" ]; then
log_warn "Package directory not found: ${pkg_dir}"
continue
fi
# Update version in control file
control_file="${pkg_dir}/DEBIAN/control"
if [ -f "${control_file}" ]; then
sed -i "s/^Version:.*/Version: ${VERSION}/" "${control_file}"
fi
# Build the package
cd "${PACKAGING_DIR}"
dpkg-deb --build --root-owner-group "${pkg}" "${OUTPUT_DIR}/${pkg}_${VERSION}_amd64.deb"
# Validate the package
log_info "Validating package..."
dpkg-deb -I "${OUTPUT_DIR}/${pkg}_${VERSION}_amd64.deb"
dpkg-deb -c "${OUTPUT_DIR}/${pkg}_${VERSION}_amd64.deb" | head -20
echo ""
done
log_info "All packages built successfully!"
log_info "Output directory: ${OUTPUT_DIR}"
ls -la "${OUTPUT_DIR}"