Files
qt5-legatus-legacy/Lts/Tools/DBAdministration/copy_exercise.cpp
T
2026-08-13 18:33:10 +02:00

634 lines
21 KiB
C++
Executable File

/*
* @RDE_HEADER@
* Copy exercise from server to evaluation server
* $Id: copy_exercise.cpp 64999 2025-07-01 04:02:56Z fritsche_h $
* $HeadURL: svn://192.168.133.21/TDS/branches/product/release_candidates/guez/3dws_rc/Lts/Tools/DBAdministration/copy_exercise.cpp $
* $Author: fritsche_h $
* Created by Christian Ehrlicher, NLSE, 2018-07-31
*/
#include "copy_exercise.h"
#include <ngldb/dbcommon.h>
#include <ngldb/dbconnection.h>
#include <ngldb/nsqlquery.h>
#include <ngltools/ngltools_filesystem.h>
#include <QProcess>
#include <QFile>
#include <QDir>
#include <QSqlError>
#include <QVariant>
namespace DBAdministration
{
CopyExercise::CopyExercise(NGL::INGLLogger &logger, CopyFor copyFor, QObject *parent)
: QObject(parent)
, m_proc(nullptr)
, m_procPing(nullptr)
, m_exerciseId(-1)
, m_copyFor(copyFor)
, m_timer(new QTimer(this))
, m_stopPing(false)
, m_firstPing(false)
, m_progressBarValue(0)
, m_logger(logger)
{
QStringList args({QStringLiteral("-W 10"), // QStringLiteral("10"),
QStringLiteral("-c 6"), //QStringLiteral("6"),
QStringLiteral("EXCDBS01")});
handleProcPing(args);
}
CopyExercise::~CopyExercise()
{
}
bool CopyExercise::init(int exerciseId,
const QString &srcDbHost, const QString &srcDbName,
const QString &dstDbHost, const QString &dstDbName)
{
if (m_exerciseId > 0 )
return true;
m_exerciseId = exerciseId;
m_srcDbName = srcDbName;
m_srcDbHost = srcDbHost;
m_dstDbHost = dstDbHost;
m_dstDbName = dstDbName;
// Server Databases
m_dbSrv = DBConnection::open(m_logger,
QStringLiteral("ctc_sv"),
QStringLiteral("ctc11de"),
QStringLiteral("QPSQL"),
m_srcDbHost,
m_srcDbName,
QString(), // <-- default
m_srcDbHost + QLatin1Char('_') + m_srcDbName + QLatin1Char('_') + QString::number(m_exerciseId));
if (m_copyFor == CopyForEvaluation)
{
m_dbEvaluation = DBConnection::open(m_logger,
QStringLiteral("ctc_sv"),
QStringLiteral("ctc11de"),
QStringLiteral("QPSQL"),
m_dstDbHost,
m_dstDbName,
QString(),
m_dstDbHost + QLatin1Char('_') + m_dstDbName + QLatin1Char('_') + QString::number(m_exerciseId));
if (!m_dbEvaluation.isOpen())
{
WARNING_LOG(m_logger) << "Could not open DB connection to " << m_dstDbHost
<< ", error: " << m_dbEvaluation.lastError().databaseText()
<< ", " << m_dbEvaluation.lastError().driverText();
return false;
}
}
else if (m_copyFor == CopyForAAR)
{
// Only open exercise for creating exercise user
m_dbEvaluation = DBConnection::open(m_logger,
QStringLiteral("postgres"),
QStringLiteral("postgres"),
QStringLiteral("QPSQL"),
m_dstDbHost,
QStringLiteral("postgres"),
QString(),
m_dstDbHost + QLatin1Char('_') + QStringLiteral("postgres") + QLatin1Char('_') + QString::number(m_exerciseId));
if (!m_dbEvaluation.isOpen())
{
WARNING_LOG(m_logger) << "Could not open DB connection to " << m_dstDbHost
<< ", error: " << m_dbEvaluation.lastError().databaseText()
<< ", " << m_dbEvaluation.lastError().driverText();
}
else
{
NSqlQuery q(m_dbEvaluation);
//QString user = QStringLiteral("ets_ex_") + QString::number(m_exerciseId);
QString user = QStringLiteral("ets_ex_00");
QString stmt = QStringLiteral("SELECT create_user_if_not_exists('") + user + QStringLiteral("')");
if (!q.exec(stmt))
ERROR_LOG(m_logger) << "Error while executing stmt: " << stmt << " error: " << m_dbEvaluation.lastError().databaseText();
else
INFO_LOG(m_logger) << "Created User " << user;
m_dbEvaluation.close();
}
}
if (!m_dbSrv.isOpen())
{
WARNING_LOG(m_logger) << "Could not open DB connection to " << m_srcDbHost
<< ", error: " << m_dbSrv.lastError().databaseText()
<< ", " << m_dbSrv.lastError().driverText();
return false;
}
DEBUG_LOG(m_logger) << "Successfully initialized database connections";
return true;
}
void CopyExercise::setDownloadScriptIds(const QVector<int> &downloadScriptIds)
{
m_downloadScriptIds = downloadScriptIds;
}
bool CopyExercise::doCopy(int exerciseId)
{
m_firstPing = true;
onPingStart();
if (exerciseId > 0)
m_exerciseId = exerciseId;
return true;
}
void CopyExercise::dropDb()
{
const QString dbName = QStringLiteral("lts_db_") + QString::number(m_exerciseId);
INFO_LOG(m_logger) << "Database to be dropped : " << dbName;
const QStringList args({QStringLiteral("-U"),
QStringLiteral("postgres"),
QStringLiteral("-h"),
QStringLiteral("localhost"),
dbName});
handleProcInit(QStringLiteral("dropdb"), args);
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onDropDbFinished2);
m_proc->start();
}
bool CopyExercise::runPgDump()
{
const QString schema = DBCommon::getExerciseSchemaName(m_exerciseId);
QString stmt = QStringLiteral("select sum(pg_total_relation_size(quote_ident(schemaname) || '.' || quote_ident(tablename)))::bigint from pg_tables where schemaname IN ('") + schema +
QStringLiteral("','exm','asm', 'avm', 'bdv', 'core', 'flm', 'aes', 'ism', 'mdm', 'msg', 'opm', 'ppm', 'srm', 'gna', 'tab', 'whg')");
DEBUG_LOG(m_logger) << "stmt: " << stmt;
NSqlQuery q(m_dbSrv);
if (!q.execAndFetch(stmt))
{
ERROR_LOG(m_logger) << "Error while estimating size of database dump errror: " << m_dbSrv.lastError().databaseText();
m_stopPing = true;
return false;
}
m_sizePgDump = q.value(0).toLongLong();
double seconds = m_sizePgDump / 1024 / 100 * 0.0015;
INFO_LOG(m_logger) << "Size database dump: " << m_sizePgDump;
INFO_LOG(m_logger) << " seconds: " << seconds;
m_timer->setInterval(seconds * 1000);
connect(m_timer, &QTimer::timeout, this, &CopyExercise::onTimeoutTimer);
// start process
const QStringList args({
QStringLiteral("-n"), schema,
QStringLiteral("-n"), QStringLiteral("public"),
QStringLiteral("-n"), QStringLiteral("exm"),
QStringLiteral("-n"), QStringLiteral("asm"),
QStringLiteral("-n"), QStringLiteral("avm"),
QStringLiteral("-n"), QStringLiteral("bdv"),
QStringLiteral("-n"), QStringLiteral("core"),
QStringLiteral("-n"), QStringLiteral("flm"),
QStringLiteral("-n"), QStringLiteral("aes"),
QStringLiteral("-n"), QStringLiteral("ism"),
QStringLiteral("-n"), QStringLiteral("mdm"),
QStringLiteral("-n"), QStringLiteral("msg"),
QStringLiteral("-n"), QStringLiteral("opm"),
QStringLiteral("-n"), QStringLiteral("pdm"),
QStringLiteral("-n"), QStringLiteral("ppm"),
QStringLiteral("-n"), QStringLiteral("srm"),
QStringLiteral("-n"), QStringLiteral("gna"),
QStringLiteral("-n"), QStringLiteral("tab"),
QStringLiteral("-n"), QStringLiteral("whg"),
QStringLiteral("-U"), QStringLiteral("postgres"),
QStringLiteral("-Fc"),
QStringLiteral("-f"), getDumpFile(),
QStringLiteral("-h"), m_srcDbHost,
m_srcDbName
});
handleProcInit(QStringLiteral("pg_dump"), args);
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onPgDumpFinished);
m_proc->start();
m_progressBarValue = 1;
m_timer->start();
return true;
}
void CopyExercise::onTimeoutTimer()
{
INFO_LOG(m_logger) << "in onTimeoutTimer()";
m_progressBarValue += 1;
INFO_LOG(m_logger) << "emit : " << m_progressBarValue;
emit nextProgress(m_progressBarValue);
}
void CopyExercise::onPgDumpFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
m_timer->stop();
// 1/4 done
m_progressBarValue = 25;
emit nextProgress(m_progressBarValue);
if (!checkExitStatus(QStringLiteral("onPgDumpFinished"), exitCode, exitStatus))
{
QFile::remove(getDumpFile());
dropDb();
return;
}
const QString dbName = QStringLiteral("lts_db_") + QString::number(m_exerciseId);
const QStringList args({QStringLiteral("-U"),
QStringLiteral("postgres"),
QStringLiteral("-h"),
m_dstDbHost,
dbName});
handleProcInit(QStringLiteral("dropdb"), args);
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onDropDbFinished);
m_proc->start();
}
void CopyExercise::onDropDbFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
INFO_LOG(m_logger) << "onDropDbFinished: exitCode: " << exitCode << ", exitStatus: " << exitStatus;
m_progressBarValue = 30;
emit nextProgress(m_progressBarValue);
const QString dbName = QStringLiteral("lts_db_") + QString::number(m_exerciseId);
const QStringList args({QStringLiteral("-U"),
QStringLiteral("postgres"),
QStringLiteral("-T"),
QStringLiteral("template2"),
QStringLiteral("-h"),
m_dstDbHost,
dbName});
handleProcInit(QStringLiteral("createdb"), args);
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onCreateDbFinished);
m_proc->start();
}
void CopyExercise::onDropDbFinished2(int exitCode, QProcess::ExitStatus exitStatus)
{
ERROR_LOG(m_logger) << "onDropDbFinished2: exitCode: " << exitCode << ", exitStatus: " << exitStatus;
emit copyDbFailed();
}
void CopyExercise::onCreateDbFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
INFO_LOG(m_logger) << "onCreateDbFinished: exitCode: " << exitCode << ", exitStatus: " << exitStatus;
m_progressBarValue = 35;
emit nextProgress(m_progressBarValue);
if (exitCode > 0)
{
ERROR_LOG(m_logger) << "Exercise " << m_exerciseId << " already exists";
QFile::remove(getDumpFile());
emit copyDbFailed();
return;
}
const QString dbName = QStringLiteral("lts_db_") + QString::number(m_exerciseId);
const QStringList args({QStringLiteral("-U"),
QStringLiteral("postgres"),
QStringLiteral("-h"),
m_dstDbHost,
QStringLiteral("-d"),
dbName,
QStringLiteral("-j"),
QStringLiteral("4"),
getDumpFile(),
});
handleProcInit(QStringLiteral("pg_restore"), args);
if (m_copyFor == CopyForEvaluation)
{
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onPsqlFinishedForEvaluation);
}
else
{
connect(m_proc, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished),
this, &CopyExercise::onPsqlFinishedForAAR);
}
double seconds = m_sizePgDump / 1024 / 100 * 0.001;
INFO_LOG(m_logger) << " seconds: " << seconds;
m_timer->setInterval(seconds * 1000);
m_timer->start();
m_proc->start();
}
void CopyExercise::onPsqlFinishedForEvaluation(int exitCode, QProcess::ExitStatus exitStatus)
{
m_timer->stop();
m_progressBarValue = 75;
emit nextProgress(m_progressBarValue);
if (!checkExitStatus(QStringLiteral("onPsqlFinishedForEvaluation"), exitCode, exitStatus))
return;
if ( exitStatus == QProcess::ExitStatus::NormalExit )
QFile::remove(getDumpFile());
// get exercise data from db
QVariant exName, exStartTime, exEndTime;
{
NSqlQuery q(m_dbSrv);
QString stmt = QStringLiteral("SELECT name, begin_time, end_time FROM exm.exercise WHERE id = ") + QString::number(m_exerciseId);
if (q.execAndFetch(stmt))
{
int i = 0;
exName = q.value(i++);
exStartTime = q.value(i++);
exEndTime = q.value(i++);
}
else
{
ERROR_LOG(m_logger) << "Error while reading exercise info for exercise " << m_exerciseId << " from source database"
<< ", error: " << q.lastError().databaseText()
<< ", " << q.lastError().driverText()
<< ", stmt: " << stmt;
emit copyDbFailed();
m_stopPing = true;
return;
}
}
NSqlQuery q(m_dbEvaluation);
QString stmt = QStringLiteral("INSERT INTO local_databases (name, db_name, begin_time, end_time) "
"VALUES (:name, :dbName, :beginTime, :endTime) ");
q.prepare(stmt);
q.bindValue(QStringLiteral(":name"), exName);
QString dbName = QStringLiteral("lts_db_") + QString::number(m_exerciseId);
q.bindValue(QStringLiteral(":dbName"), dbName);
q.bindValue(QStringLiteral(":beginTime"), exStartTime);
q.bindValue(QStringLiteral(":endTime"), exEndTime);
if (!q.exec())
{
ERROR_LOG(m_logger) << "Error while inserting data in local_databases: exerciseId = " << m_exerciseId
<< ", error: " << q.lastError().databaseText()
<< ", " << q.lastError().driverText();
emit copyDbFailed();
}
else
{
emit copyDbFinished();
}
m_stopPing = true;
m_dbEvaluation.close();
}
void CopyExercise::onPsqlFinishedForAAR(int exitCode, QProcess::ExitStatus exitStatus)
{
m_timer->stop();
m_progressBarValue = 75;
emit nextProgress(m_progressBarValue);
if (!checkExitStatus(QStringLiteral("onPsqlFinishedForAAR"), exitCode, exitStatus))
return;
if ( exitStatus == QProcess::ExitStatus::NormalExit )
QFile::remove(getDumpFile());
//init procedures in schema public
const QString fileName = QStringLiteral("/shd/CTC/AAR/bin/test/init_db_procedures.sql");
const QStringList args({QStringLiteral("-U"),
QStringLiteral("postgres"),
QStringLiteral("-h"),
m_dstDbHost,
QStringLiteral("-d"),
m_dstDbName,
QStringLiteral("-f"),
fileName
});
handleProcInit(QStringLiteral("psql"), args);
//init procedures in schema public
// now open the downloaded database
if (!m_dbEvaluation.isOpen())
{
m_dbEvaluation = DBConnection::open(m_logger,
QStringLiteral("postgres"),
QStringLiteral("postgres"),
QStringLiteral("QPSQL"),
m_dstDbHost,
m_dstDbName,
QString(),
m_dstDbHost + QLatin1Char('_') + m_dstDbName + QLatin1Char('_') + QString::number(m_exerciseId));
if (!m_dbEvaluation.isOpen())
{
ERROR_LOG(m_logger) << "Could not open DB connection to " << m_dstDbHost
<< ", error: " << m_dbEvaluation.lastError().databaseText()
<< ", " << m_dbEvaluation.lastError().driverText();
emit copyDbFailed();
m_stopPing = true;
return;
}
}
NSqlQuery q(m_dbEvaluation);
const QString schema = DBCommon::getExerciseSchemaName(m_exerciseId);
QStringList downloadScriptIds;
downloadScriptIds.reserve(m_downloadScriptIds.size());
for (int id : qAsConst(m_downloadScriptIds))
downloadScriptIds.push_back(QString::number(id));
QString stmt = QLatin1String("DELETE FROM ") + schema + QLatin1String(".pre_presentation WHERE id not in (") + downloadScriptIds.join(QLatin1Char(',')) + QLatin1Char(')');
if (!q.exec(stmt))
{
ERROR_LOG(m_logger) << "Error while deleting scripts from database";
emit copyDbFailed();
}
else
{
// stmt = QLatin1String("DELETE FROM exm.exercise WHERE id != ") + QString::number(m_exerciseId);
// deleting foreign key will delete all dependences in cascade, so all FLM data for that exercise
stmt = QLatin1String("UPDATE exm.exercise SET exercise_state_id = 6 WHERE id != ") + QString::number(m_exerciseId);
DEBUG_LOG(m_logger) << "Executing statement: " << stmt;
if (!q.exec(stmt))
ERROR_LOG(m_logger) << "Error on updating exm.exercise " << m_dbEvaluation.lastError().databaseText();
else
DEBUG_LOG(m_logger) << "Statement succesfully executed";
stmt = QLatin1String("GRANT USAGE ON SCHEMA PUBLIC TO ets_ro;");
DEBUG_LOG(m_logger) << "Executing statement: " << stmt;
if (!q.exec(stmt))
ERROR_LOG(m_logger) << "Error on grant usage " << m_dbEvaluation.lastError().databaseText();
else
DEBUG_LOG(m_logger) << "Statement succesfully executed";
stmt = QLatin1String("GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA public TO ets_ro;");
DEBUG_LOG(m_logger) << "Executing statement: " << stmt;
if (!q.exec(stmt))
ERROR_LOG(m_logger) << "Error on grant execute " << m_dbEvaluation.lastError().databaseText();
else
DEBUG_LOG(m_logger) << "Statement succesfully executed";
emit copyDbFinished();
}
m_stopPing = true;
m_dbEvaluation.close();
}
bool CopyExercise::checkExitStatus(const QString &procName, int exitCode, QProcess::ExitStatus exitStatus)
{
if (exitCode > 0)
{
NOTICE_LOG(m_logger) << procName << ": exitCode: " << exitCode << ", exitStatus: " << exitStatus;
WARNING_LOG(m_logger) << m_proc->readAllStandardError() << m_proc->readAllStandardOutput();
m_stopPing = true;
return false;
}
return true;
}
void CopyExercise::handleProcPing(const QStringList &args)
{
if (m_procPing)
{
m_procPing->disconnect(this);
m_procPing->deleteLater();
}
m_procPing = new QProcess(this);
QString errStr;
const QString exePath = NGL::postgresExecutablePath(QStringLiteral("ping"), errStr);
if (!errStr.isEmpty())
WARNING_LOG(m_logger) << errStr;
INFO_LOG(m_logger) << "Starting process '" << exePath << "' with args '"
<< args.join(QLatin1Char(' ')) << "'";
m_procPing->setProgram(exePath);
m_procPing->setArguments(args);
connect(m_procPing, &QProcess::errorOccurred,
this, &CopyExercise::onErrorOccured);
connect(m_procPing, QOverload<int, QProcess::ExitStatus>::of(&QProcess::finished), this, &CopyExercise::onPingFinished);
}
void CopyExercise::onPingStart()
{
m_stopPing = false;
m_procPing->start();
INFO_LOG(m_logger) << "ping started ";
}
void CopyExercise::onPingFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
if (exitCode > 0)
DEBUG_LOG(m_logger) << "got ping result with exitCode != 0. exitCode: " << exitCode << ", exitStatus: " << exitStatus;
else
{
if (m_firstPing)
{
m_firstPing = false;
if (!runPgDump())
{
ERROR_LOG(m_logger) << "Error running pg_dump for exercise " << m_exerciseId;
emit copyDbFailed();
m_stopPing = true;
}
}
if (!m_stopPing)
QTimer::singleShot(2000, this, &CopyExercise::onPingStart);
}
}
void CopyExercise::stopPing()
{
m_stopPing = true;
}
void CopyExercise::handleProcInit(const QString &program, const QStringList &args)
{
if (m_proc)
{
m_proc->disconnect(this);
m_proc->deleteLater();
}
m_proc = new QProcess(this);
QString errStr;
const QString exePath = NGL::postgresExecutablePath(program, errStr);
if (!errStr.isEmpty())
WARNING_LOG(m_logger) << errStr;
INFO_LOG(m_logger) << "Starting process '" << exePath << "' with args '"
<< args.join(QLatin1Char(' ')) << "'";
m_proc->setProgram(exePath);
m_proc->setArguments(args);
connect(m_proc, &QProcess::errorOccurred,
this, &CopyExercise::onErrorOccured);
}
void CopyExercise::handleProcFinished(int exitCode, QProcess::ExitStatus exitStatus)
{
if (!m_proc)
return;
if (exitCode != 0 || exitStatus != 0)
{
QByteArray out = m_proc->readAllStandardError();
if (!out.isEmpty())
out += '\n';
out += m_proc->readAllStandardOutput();
ERROR_LOG(m_logger) << "Process finished - exitCode: " << exitCode << ", exitStatus = " << exitStatus << ", " << out;
}
m_proc->disconnect(this);
m_proc->deleteLater();
m_proc = nullptr;
}
void CopyExercise::onErrorOccured(QProcess::ProcessError error)
{
ERROR_LOG(m_logger) << "onErrorOccured: error: " << error << ", " << m_proc->readAllStandardOutput() << ", " << m_proc->readAllStandardError();
m_timer->stop();
}
QString CopyExercise::getDumpFile() const
{
return QDir::tempPath() + QLatin1Char('/') + m_srcDbName + QLatin1Char('_')+ QString::number(m_exerciseId) + QStringLiteral(".dump");
}
} // namespace DBAdministration