Initial commit

Lts/Services/RecordingManagement/CMakeLists.txt

@@ -0,0 +1,125 @@
+#
+# $Id$
+# $HeadURL$
+# $Author$
+#
+
+INCLUDE(MacroRTIDDSGen)
+INCLUDE(MacroCodeGenIdl)
+
+SET(xml_FILE rec_service.xml)
+
+#
+# --- Library svcRecCommon --------------------------------------------------------------------------------
+#
+SET(SRCS_common
+  rec_export.h
+  rec.hpp # for rec_json.cpp
+
+  rec_db_impl_custom.cpp
+  rec_db_impl_custom.h
+  logger.cpp
+  logger.h
+)
+
+# Generate rep_enums.idl / rep.idl
+CODEGEN_IDL_SVC(SRCS_common ${xml_FILE})
+# Generate rec_db_impl.cpp/.h, rec_db.h
+CODEGEN_DATABASE(SRCS_common ${xml_FILE})
+
+# Generate *.cpp/*.hpp from idl
+RTIDDSGEN(SRCS_common rec_enums.idl)
+
+# Generate rec_dt[s,c]_types.h, rec_ft[s,c]_types.h
+CODEGEN_TOPIC_TYPEDEFS(SRCS_common ${xml_FILE})
+
+NGL_ADD_LIBRARY(AAR SV WS svcRecCommon SHARED  ${SRCS_common})
+NGL_TARGET_INCLUDE_DIRECTORIES(svcRecCommon
+  PUBLIC
+    ${CMAKE_CURRENT_SOURCE_DIR}/../
+    ${CMAKE_CURRENT_BINARY_DIR}/../
+)
+NGL_TARGET_LINK_LIBRARIES(svcRecCommon
+  PRIVATE
+    RTI::nddscpp2
+  PUBLIC
+    Qt5::Core
+    ngldb
+    LtsUtilsCore
+    ddscore
+    NGLLogger
+)
+NGL_INSTALL_TARGET(svcRecCommon)
+
+#
+# --- Library svcRecServer --------------------------------------------------------------------------------
+#
+#SET(SRCS_server
+#  rec_export.h
+#)
+
+#NGL_ADD_LIBRARY(AAR SV svcRecServer SHARED ${SRCS_server})
+#NGL_TARGET_INCLUDE_DIRECTORIES(svcRecServer
+#  PUBLIC
+#    ${CMAKE_CURRENT_SOURCE_DIR}/../
+#    ${CMAKE_CURRENT_BINARY_DIR}/../
+#)
+
+#NGL_TARGET_LINK_LIBRARIES(svcRecServer
+#  PRIVATE
+#    RTI::nddscpp2
+#    ddsconnector
+#    ddsfts
+#  PUBLIC
+#    svcRecCommon
+#    ngldb
+#    LtsUtilsCore
+#)
+#NGL_INSTALL_TARGET(svcRecServer)
+
+#
+# --- Executable svcRecordingManagement (sREC) -------------------------------------------------
+#
+SET(SRCS_exe
+  db_writer.cpp
+  db_writer.h
+  db_writer_snapshot.cpp
+  db_writer_snapshot.h
+  dds_handler.cpp
+  dds_handler.h
+  main_handler.cpp
+  main_handler.h
+  rec_export.h
+  main.cpp
+  settings.cpp
+  settings.h
+  ${xml_FILE}
+)
+
+# Generate rec.sql, rec.xml
+CODEGEN_SQL(SRCS_exe ${xml_FILE})
+
+NGL_ADD_SVC_ICON(SRCS_exe)
+NGL_ADD_EXECUTABLE(AAR SV svcRecordingManagement ${SRCS_exe})
+NGL_TARGET_LINK_LIBRARIES(svcRecordingManagement
+  Qt5::Core
+  svcRecCommon
+ # svcRecServer
+  svcCoreClient
+  svcExmCommon
+  ddsconnector
+  nglsettings
+  ProjectConfig
+  LtsCommandLineParser
+  NGLLogger
+  nglcoreapplication
+  LtsPluginLoader
+  ngltools
+)
+NGL_INSTALL_TARGET(svcRecordingManagement)
+NGL_INSTALL(AAR SV FILES log4cxx_rec.props DESTINATION bin)
+NGL_ADD_TARGET_LINK(svcRecordingManagement sREC)
+
+# Uncomment if necessary
+#ADD_SUBDIRECTORY(UnitTests)
+#ADD_SUBDIRECTORY(ModuleTests)

Lts/Services/RecordingManagement/db_writer.cpp

@@ -0,0 +1,136 @@
+/**
+  * DbWriter for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#include "db_writer.h"
+
+#include "db_writer_snapshot.h"
+#include "rec_db.h"
+#include "logger.h"
+#include "settings.h"
+#include <DDS/DDSConnector/dds_connector.h>
+
+#include <QThread>
+
+#include <ngldb/dbtransaction.h>
+
+#include <LtsPluginManager/lts_plugin_loader.h>
+#include <RecordingManagement/rec_db_impl.h>
+
+namespace RecordingManagement
+{
+
+DbWriter::DbWriter(const std::shared_ptr<DDSConnector::DdsConnector> &ddsConnector)
+  : m_query(nullptr)
+  , m_recordTimer(new QTimer(this))
+  , m_ddsConnector(ddsConnector)
+  , m_dbWriterSnapshot(nullptr)
+  , m_dbWriterSnapshotThread(nullptr)
+  , m_logger(getLogger("DbWriter"))
+{}
+
+DbWriter::~DbWriter()
+{
+  if (m_dbWriterSnapshotThread)
+  {
+    m_dbWriterSnapshotThread->quit();
+    m_dbWriterSnapshotThread->wait(30 * 1000);
+    delete m_dbWriterSnapshotThread;
+  }
+  LtsPluginManager::LtsPluginLoader::getInstance()->unloadAllPlugins();
+  m_plugins.clear();
+
+  delete m_dbWriterSnapshot;
+  delete m_recordTimer;
+  delete m_query;
+}
+
+bool DbWriter::loadPlugin(const LtsPluginManager::PluginInstance &pi)
+{
+  RecordReplay::RecordReplayPlugin *recPlugin = qobject_cast<RecordReplay::RecordReplayPlugin *>(pi.plugin);
+  if (!recPlugin)
+  {
+    ERROR_LOG(m_logger) << "Cannot load RecordReplayPlugin " << pi.fileName;
+    return false;
+  }
+  INFO_LOG(m_logger) << "Loading RecordReplayPlugin " << pi.fileName;
+  // exId == partId da wir hier in IDL::PartitionType::Exercise arbeiten
+  if (!recPlugin->initDtc(m_ddsConnector, m_ddsConnector->getPartitionId()))
+  {
+    ERROR_LOG(m_logger) << "Initialisation for RecordReplayPlugin " << pi.fileName << " failed ";
+    return false;
+  }
+
+  INFO_LOG(m_logger) << "Loaded record replay plugin " << recPlugin->pluginName();
+  m_plugins += recPlugin;
+
+  return true;
+}
+
+bool DbWriter::init(const std::shared_ptr<DatabaseInterface> &db)
+{
+  m_db = db;
+
+  auto cb = [this](const LtsPluginManager::PluginInstance &pi) { return loadPlugin(pi);};
+  LtsPluginManager::LtsPluginLoader::getInstance()->loadPlugins(LtsPluginManager::PluginType::SPECIAL,
+                                                                cb,
+                                                                QStringList(),
+                                                                QStringLiteral("recordreplay"));
+
+  // Start recording data
+  qint64 msecs = Settings::updateFrequency();
+  INFO_LOG(m_logger) << "Updating recording tables every " << msecs << " milliseconds.";
+  connect(m_recordTimer, &QTimer::timeout, this, &DbWriter::ddsTopicsToDatabase);
+  m_recordTimer->start(msecs);
+
+  delete m_query;
+  m_query = new NSqlQuery(m_db->database());
+
+  return initSnapshot();
+}
+
+bool DbWriter::initSnapshot()
+{
+  if (m_dbWriterSnapshotThread)
+  {
+    m_dbWriterSnapshotThread->quit();
+    m_dbWriterSnapshotThread->wait(30 * 1000);
+    delete m_dbWriterSnapshotThread;
+  }
+  delete m_dbWriterSnapshot;
+
+  m_dbWriterSnapshot = new DbWriterSnapshot(m_plugins,
+                                            m_ddsConnector->getPartitionId(),
+                                            Settings::settings());
+  m_dbWriterSnapshotThread = new DbWriterSnapshotThread(m_dbWriterSnapshot, this);
+  m_dbWriterSnapshot->moveToThread(m_dbWriterSnapshotThread);
+  m_dbWriterSnapshotThread->start();
+
+  return true;
+}
+
+void DbWriter::ddsTopicsToDatabase()
+{
+  DEBUG_LOG(m_logger) << "IN ddsTopicsToDatabase";
+  DBTransaction trans(m_db->database());
+
+  QHash<QString, qint64> minRecordSampleTimes;
+
+  // for each plugin
+  for (RecordReplay::RecordReplayPlugin *recPlugin : qAsConst(m_plugins))
+  {
+    if (!recPlugin->ddsTopicsToDatabase(*m_query, minRecordSampleTimes))
+    {
+      // should never happen but who knows
+      ERROR_LOG(m_logger) << "Error recording dds topics - aborting recording data";
+      return;
+    }
+  }
+  trans.commit();
+
+  m_dbWriterSnapshot->setMinRecordedSampleTimes(minRecordSampleTimes);
+}
+
+}  // RecordingManagement

Lts/Services/RecordingManagement/db_writer.h

@@ -0,0 +1,55 @@
+/**
+  * DbWriter for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#pragma once
+
+#include <QObject>
+#include <QTimer>
+#include <QVector>
+
+#include <PluginInterfaces/record_replay_plugin.h>
+#include <ngldb/nsqlquery.h>
+
+#include "logger.h"
+
+namespace LtsPluginManager
+{
+struct PluginInstance;
+}
+namespace RecordingManagement
+{
+class DatabaseInterface;
+class DbWriterSnapshot;
+class DbWriterSnapshotThread;
+
+class DbWriter : public QObject
+{
+  Q_OBJECT
+public:
+  explicit DbWriter(const std::shared_ptr<DDSConnector::DdsConnector> &ddsConnector);
+  ~DbWriter();
+
+  bool init(const std::shared_ptr<DatabaseInterface> &db);
+
+protected:
+  bool initSnapshot();
+  bool loadPlugin(const LtsPluginManager::PluginInstance &pi);
+
+private Q_SLOTS:
+  void ddsTopicsToDatabase();
+
+private:
+  NSqlQuery                                  *m_query{nullptr};
+  QTimer                                     *m_recordTimer{nullptr};
+  std::shared_ptr<DDSConnector::DdsConnector> m_ddsConnector;
+  QVector<RecordReplay::RecordReplayPlugin *> m_plugins;
+  DbWriterSnapshot                           *m_dbWriterSnapshot{nullptr};
+  DbWriterSnapshotThread                     *m_dbWriterSnapshotThread{nullptr};
+  std::shared_ptr<DatabaseInterface>         m_db;
+
+  LOGGER_T m_logger;
+};
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/db_writer_snapshot.cpp

@@ -0,0 +1,306 @@
+/**
+  * DbWriterSnapshot for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#include "db_writer_snapshot.h"
+
+#include "logger.h"
+
+#include <QSqlError>
+#include <QTimer>
+#include <QVariant>
+
+#include <nglsettings/nglsettings_settings.h>
+#include <ngldb/dbtransaction.h>
+#include <ngldb/nsqlquery.h>
+
+#include <PluginInterfaces/record_replay_plugin.h>
+#include <RecordingManagement/rec_db_impl.h>
+
+namespace RecordingManagement
+{
+
+void DbWriterSnapshotThread::run()
+{
+  if (!m_dbWriterSnapshot)
+    return;
+  m_dbWriterSnapshot->init();
+  // make sure m_dbWriterSnapshot lives in our thread
+  Q_ASSERT(m_dbWriterSnapshot->thread() == this);
+  exec();
+  m_dbWriterSnapshot->cleanup();
+}
+
+DbWriterSnapshot::DbWriterSnapshot(const QVector<RecordReplay::RecordReplayPlugin *> &plugins,
+                                   int exerciseId,
+                                   NGL::Settings *settings)
+  : QObject()
+  , m_plugins(plugins)
+  , m_snapshotInterval(1 * 60 * 1000) // TODO: make config variable   // Interval: 1 min in milliseconds
+  , m_exerciseId(exerciseId)
+  , m_settings(settings)
+  , m_logger(getLogger("DbWriterSnapshot"))
+{
+}
+
+DbWriterSnapshot::~DbWriterSnapshot()
+{
+  cleanup();  // should do nothing because cleanup() should be called from within the thread
+}
+
+void DbWriterSnapshot::init()
+{
+  m_dbInterface = DatabaseInterface::create(m_exerciseId);
+
+  QSqlDatabase dbConn = QSqlDatabase::addDatabase(m_settings->databaseDriver(),
+                                                  dbConnectionName());
+  dbConn.setHostName(m_settings->databaseHostname());
+  dbConn.setDatabaseName(m_settings->databaseName());
+  dbConn.setUserName(m_dbInterface->dbUser());
+  dbConn.setPassword(m_settings->databasePassword());
+
+  if (!dbConn.open())
+  {
+    FATAL_LOG(m_logger) <<  "Error opening database " << m_settings->databaseHostname()
+                         <<  ":" << m_settings->databaseName()
+                          << " as user " << m_dbInterface->dbUser();
+    FATAL_LOG(m_logger) << dbConn.lastError().driverText() << " "
+                        << dbConn.lastError().databaseText() << " "
+                        << dbConn.lastError().nativeErrorCode();
+    m_dbInterface = nullptr;
+    return;
+  }
+
+
+  // Set search path
+  const QString stmt = QLatin1String("SET SEARCH_PATH TO ") + m_dbInterface->schemaName() + QLatin1String(", PUBLIC");
+  INFO_LOG(m_logger) << "SearchPath Stmt: " << stmt;
+  NSqlQuery q(dbConn);
+
+  q.exec(stmt);
+
+  if (!m_dbInterface->init(dbConn))
+  {
+    ERROR_LOG(m_logger) << "Error initializing database interface - can not initialize DbWriterSnapshot";
+    m_dbInterface = nullptr;
+    return;
+  }
+
+  QTimer::singleShot(0, Qt::VeryCoarseTimer,
+                     this, &DbWriterSnapshot::initSnapshot);
+}
+
+void DbWriterSnapshot::cleanup()
+{
+  m_dbInterface = nullptr;
+  QSqlDatabase::database(dbConnectionName()).close();
+}
+
+void DbWriterSnapshot::setMinRecordedSampleTimes(const QHash<QString, qint64> &minRecordedSampleTimes)
+{
+  QMutexLocker l(&m_minRecordedSampleTimesMutex);
+  for(auto it = minRecordedSampleTimes.begin(); it != minRecordedSampleTimes.end(); ++it)
+  {
+    const qint64 minRecordedSampleTime = it.value();
+    const QString &topicName = it.key();
+    if (minRecordedSampleTime == 0)
+    {
+      WARNING_LOG(m_logger) << "Minimum recorded Sample Time for topic " << topicName <<
+                               " is zero - looks like a service did not properly set the sample time!";
+      continue;
+    }
+    auto mrstIt = m_minRecordedSampleTimes.constFind(topicName);
+    if (mrstIt == m_minRecordedSampleTimes.constEnd())
+    {
+      // no data until now - just take minRecordedSampleTime
+      m_minRecordedSampleTimes[topicName] = minRecordedSampleTime;
+    }
+    else
+    {
+      // already a minimum time - make sure to get the minimum from both
+      m_minRecordedSampleTimes[topicName] = std::min(minRecordedSampleTime, mrstIt.value());
+    }
+    INFO_LOG(m_logger) << "Current minimum sample time for " << topicName
+                       << " is " << m_minRecordedSampleTimes.value(topicName)
+                       << " = " << QDateTime::fromMSecsSinceEpoch(m_minRecordedSampleTimes.value(topicName)).toString();
+  }
+}
+
+
+QHash<QString, qint64> DbWriterSnapshot::getMinRecordedSampleTimes(qint64 getUntil)
+{
+  QMutexLocker l(&m_minRecordedSampleTimesMutex);
+  QHash<QString, qint64> ret;
+  // we can't remove them from m_minRecordedSampleTimes because
+  // there can be samples between getUntil and now which would not get indexed
+  // if there's no m_minRecordedSampleTimes entry for this topic
+  for (auto it = m_minRecordedSampleTimes.begin() ; it != m_minRecordedSampleTimes.end(); ++it)
+  {
+    if (it.value() < getUntil)
+    {
+      // return for recording
+      ret[it.key()] = it.value();
+      // and update minimum rec timestamp
+      *it = getUntil;
+    }
+  }
+  return ret;
+}
+
+void DbWriterSnapshot::initSnapshot()
+{
+  NSqlQuery minRecSampleTimeQuery(QSqlDatabase::database(dbConnectionName()));
+  NSqlQuery topicRecIdQuery(QSqlDatabase::database(dbConnectionName()));
+
+  topicRecIdQuery.prepare(QStringLiteral("SELECT MAX(time_stamp) FROM rec_snapshot WHERE svc_name = :svc_name AND topic_name = :topic_name"));
+
+  const qint64 tNow(time(nullptr) * 1000 - 60 * 1 * 1000); // jetzt vor einer Minute
+  INFO_LOG(m_logger) << "tNow (-1min): " << tNow;
+
+  QHash<QString, qint64> minRecordedSampleTimes;
+
+  // Recording already initialised => read last arrays for each plugin/topic to
+  // get a hint where to start with indexing each topic
+  for (RecordReplay::RecordReplayPlugin *recPlugin : qAsConst(m_plugins))
+  {
+    // get all record topics and check if there's a index for them
+    // if not -> check if there's a sample for this topic where we've to start indexing
+    const QVector<QString> recordTopicNames = recPlugin->getRecordTopicNames();
+
+    topicRecIdQuery.bindValue(QStringLiteral(":svc_name"), recPlugin->svcShortName());
+    for (const QString &topicName : recordTopicNames)
+    {
+      qint64 timeToStartSnapshotCreationFrom = 0;
+      bool bHasSnapshot = false;
+      topicRecIdQuery.bindValue(QStringLiteral(":topic_name"), topicName);
+      if (topicRecIdQuery.execAndFetch() && !topicRecIdQuery.value(0).isNull())
+      {
+        // we found a snapshot -> assume that all data until this point is properly indexed
+        // so next snapshot must be created at t + m_snapshotInterval
+        timeToStartSnapshotCreationFrom = topicRecIdQuery.value(0).toLongLong() + m_snapshotInterval;
+        bHasSnapshot = true;
+      }
+      else
+      {
+        // no snapshot for this topic. This may be because
+        // - no data until now for this topic
+        // - the index should be rebuild (== rec_snapshot entries for this topic were removed)
+        // -> check if data is available
+
+        // tNow -> default value when no data is available until now
+        timeToStartSnapshotCreationFrom = recPlugin->getMinimumSampleTime(minRecSampleTimeQuery, topicName, tNow);
+      }
+
+      const char *fromWhere = "last sample";
+      if (bHasSnapshot)
+        fromWhere = "snapshot + last sample";
+      else if (timeToStartSnapshotCreationFrom == tNow)
+        fromWhere = "current time";
+      INFO_LOG(m_logger) << "last snapshot time for topic '" << topicName
+                         << "' is: " << timeToStartSnapshotCreationFrom
+                         << " from " << fromWhere;
+      minRecordedSampleTimes[topicName] = timeToStartSnapshotCreationFrom;
+    }
+  }
+  setMinRecordedSampleTimes(minRecordedSampleTimes);
+  writeSnapshotData();
+}
+
+void DbWriterSnapshot::writeSnapshotData()
+{
+  const qint64 endTime(QDateTime::currentMSecsSinceEpoch() - 60 * 1 * 1000); // jetzt vor einer Minute
+
+  // calculate last snapshot time to create during this run
+  qint64 lastSnapshotTimeToGenerateNow = endTime - (endTime % m_snapshotInterval);
+
+  const QHash<QString, qint64> minRecSampleTimes(getMinRecordedSampleTimes(lastSnapshotTimeToGenerateNow));
+  if (minRecSampleTimes.isEmpty())
+  {
+    INFO_LOG(m_logger) << "minRecSampleTimes has no entries with time < " << lastSnapshotTimeToGenerateNow
+                       << " nothing to do...";
+    QTimer::singleShot(m_snapshotInterval, Qt::VeryCoarseTimer,
+                       this, &DbWriterSnapshot::writeSnapshotData);
+    return;
+  }
+
+  // shortcut to avoid DBTransaction and loop over all plugins
+  qint64 startTime = std::numeric_limits<qint64>::max();
+  for(const qint64 st : qAsConst(minRecSampleTimes))
+    startTime = std::min(startTime, st);
+  INFO_LOG(m_logger) << "writeSnapshotData - lastSnapshotTimeToGenerateNow: " << lastSnapshotTimeToGenerateNow
+                     << ", minRecTime is " << startTime;
+  if (startTime <= 0 || startTime > endTime)
+  {
+    WARNING_LOG(m_logger) << "Spurious start time (" << startTime
+                          << ") - using end time for start (" << endTime << ")";
+    startTime = endTime - 1;  // -1 -> create at least one interval
+  }
+
+  if (startTime > lastSnapshotTimeToGenerateNow)
+  {
+    INFO_LOG(m_logger) << "Nothing changed in the last interval until " << lastSnapshotTimeToGenerateNow << " youngest topic is at " << startTime;
+    QTimer::singleShot(m_snapshotInterval, Qt::VeryCoarseTimer,
+                       this, &DbWriterSnapshot::writeSnapshotData);
+    return;
+  }
+
+  // we know something changed between startTime and lastSnapshotTimeToGenerateNow - let's do some work
+  INFO_LOG(m_logger) << "startTime: " << startTime
+                     << " lastSnapshotTimeToGenerateNow: " << lastSnapshotTimeToGenerateNow << " - BEGIN";
+  DBTransaction trans(QSqlDatabase::database(dbConnectionName()));
+  NSqlQuery recSnapshotQuery(QSqlDatabase::database(dbConnectionName()));
+  for (RecordReplay::RecordReplayPlugin *recPlugin : qAsConst(m_plugins))
+    writeSnapshotDataPlugin(lastSnapshotTimeToGenerateNow, recSnapshotQuery, recPlugin, minRecSampleTimes);
+
+  trans.commit();
+  INFO_LOG(m_logger) << "startTime: " << startTime
+                     << " lastSnapshotTimeToGenerateNow: " << lastSnapshotTimeToGenerateNow << " - FINISHED";
+
+
+  QTimer::singleShot(m_snapshotInterval, Qt::VeryCoarseTimer,
+                     this, &DbWriterSnapshot::writeSnapshotData);
+}
+
+void DbWriterSnapshot::writeSnapshotDataPlugin(qint64 endInterval,
+                                               NSqlQuery &recSnapshotQuery,
+                                               RecordReplay::RecordReplayPlugin *recPlugin,
+                                               const QHash<QString, qint64> &minRecordedSampleTimes)
+{
+  RecordingManagement::Database::RecSnapshot recSnapshot;
+  recSnapshot.svcName = recPlugin->svcShortName();
+
+  auto snapshots =
+    recPlugin->getSnapshotTimes(recSnapshotQuery,
+                                endInterval,
+                                m_snapshotInterval,
+                                minRecordedSampleTimes);
+
+  for (auto &it : snapshots)
+  {
+    const QString &topicName = it.first;
+    auto &vec = it.second;
+
+    recSnapshot.topicName = topicName;
+    recSnapshot.timeStamp = vec.at(0).timeStamp;  // vec is sorted by timeStamp
+    m_dbInterface->removeNewerEqual(recSnapshot);
+
+    for (auto &snapshotData : vec)
+    {
+      INFO_LOG(m_logger) << "svc: " << recSnapshot.svcName
+                         << " topic: " << recSnapshot.topicName
+                         << " timestamp: " << snapshotData.timeStamp
+                         << " #elements: " << snapshotData.recIdArray.size();
+
+      recSnapshot.id = 0; // to be inserted
+      recSnapshot.timeStamp = snapshotData.timeStamp;
+      recSnapshot.recIdArray = LtsUtils::BigintArray2D(std::move(snapshotData.recIdArray));
+
+      if (!m_dbInterface->insert(recSnapshot))
+        ERROR_LOG(m_logger) << "Error while recording snaphshot for topic " << recSnapshot.topicName;
+    }
+  }
+}
+
+}  // RecordingManagement

Lts/Services/RecordingManagement/db_writer_snapshot.h

@@ -0,0 +1,83 @@
+/**
+  * DbWriter for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#pragma once
+
+#include <QHash>
+#include <QMutex>
+#include <QThread>
+#include <QVector>
+#include <memory>
+
+#include "logger.h"
+
+class NSqlQuery;
+namespace NGL
+{
+class Settings;
+}
+namespace RecordReplay
+{
+class RecordReplayPlugin;
+}
+namespace RecordingManagement
+{
+class DatabaseInterface;
+
+class DbWriterSnapshot : public QObject
+{
+  Q_OBJECT
+public:
+  explicit DbWriterSnapshot(const QVector<RecordReplay::RecordReplayPlugin *> &plugins,
+                            int exerciseId,
+                            NGL::Settings *settings);
+  ~DbWriterSnapshot();
+
+  /**
+   * Set the minimum sample time per topic which was found during the last record calls
+   * to ensure, that all telegrams are considered during snapshot creation
+   * because some telegrams can come in very later (tetra buffering / player unit out of tetra coverage)
+   */
+  void setMinRecordedSampleTimes(const QHash<QString, qint64> &minRecordedSampleTimes);
+  void init();
+  void cleanup();
+private:
+  static QString dbConnectionName() { return QStringLiteral("DbWriterSnapshotThread"); }
+  void initSnapshot();
+  QHash<QString, qint64> getMinRecordedSampleTimes(qint64 getUntil);
+  void writeSnapshotData();
+  void writeSnapshotDataPlugin(qint64 endInterval,
+                               NSqlQuery &recSnapshotQuery,
+                               RecordReplay::RecordReplayPlugin *recPlugin,
+                               const QHash<QString, qint64> &minRecordedSampleTimes);
+
+private:
+  QVector<RecordReplay::RecordReplayPlugin *> m_plugins;
+  int                                         m_snapshotInterval;
+  QHash<QString, qint64>                      m_minRecordedSampleTimes;  // do not access directly, only through minRecordedSampleTime()
+  QMutex                                      m_minRecordedSampleTimesMutex;
+  std::shared_ptr<DatabaseInterface>          m_dbInterface;
+  int                                         m_exerciseId;
+  NGL::Settings                              *m_settings;
+  LOGGER_T m_logger;
+};
+
+
+class DbWriterSnapshotThread : public QThread
+{
+  Q_OBJECT
+public:
+  DbWriterSnapshotThread(DbWriterSnapshot *dbWriterSnapshot, QObject *parent = nullptr)
+    : QThread(parent)
+    , m_dbWriterSnapshot(dbWriterSnapshot)
+  {}
+protected:
+  void run() override final;
+private:
+  DbWriterSnapshot *m_dbWriterSnapshot;
+};
+
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/dds_handler.cpp

@@ -0,0 +1,41 @@
+/**
+  * DdsHandler for DDS
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  * Created by Hanna E. Rathje, SLSE, 2016-02-03
+  */
+
+#include "dds_handler.h"
+#include <DDS/DDSConnector/subscriber.h>
+#include <DDS/Core/pacemaker.h>
+
+namespace RecordingManagement
+{
+
+DdsHandler::DdsHandler(int domainId, int exerciseId)
+  : m_ddsConnector(std::make_shared<DDSConnector::DdsConnector>(domainId, IDL::PartitionType::Exercise, exerciseId))
+  , m_logger(getLogger("DdsHandler"))
+{
+  DEBUG_LOG(m_logger) << "DdsHandler(" << domainId << ") for partition: " << m_ddsConnector->getDefaultPartition();
+}
+
+bool DdsHandler::init(const QStringList &qosFileList)
+{
+  if (!m_ddsConnector->init(qosFileList))
+  {
+    ERROR_LOG(m_logger) << "Error initializing dds connections - aborting";
+    return false;
+  }
+
+  m_ddsConnector->initPublisher();
+  // takes ownership, do not init SubscriberCallback
+  m_ddsConnector->setSubscriber(new DDSConnector::Subscriber(m_ddsConnector->getDomainParticipant(), m_ddsConnector->getDefaultPartition(), false));
+
+  // Pacemaker
+  m_pacemaker = std::make_shared<Core::Pacemaker>(m_ddsConnector);
+
+  return true;
+}
+
+} // Namespace RecordingManagement

Lts/Services/RecordingManagement/dds_handler.h

@@ -0,0 +1,42 @@
+/**
+  * DdsHandler for DDS
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  * Created by Hanna E. Rathje, SLSE, 2016-02-03
+  */
+#pragma once
+
+#include <DDS/DDSConnector/dds_connector.h>
+
+#include "logger.h"
+
+namespace Core
+{
+class Pacemaker;
+}
+
+namespace RecordingManagement
+{
+class HandlerRecordingData;
+
+class DdsHandler final
+{
+public:
+  DdsHandler(int domainId, int exerciseId);
+  ~DdsHandler() = default;
+
+  bool init(const QStringList &qosFileList = QStringList());
+
+  std::shared_ptr<Core::Pacemaker> pacemaker() const {return m_pacemaker; }
+
+  std::shared_ptr<DDSConnector::DdsConnector>          dds() const { return m_ddsConnector; }
+
+private:
+  std::shared_ptr<DDSConnector::DdsConnector>          m_ddsConnector;
+  std::shared_ptr<Core::Pacemaker>                     m_pacemaker;
+
+  NGL::INGLLogger          &m_logger;
+};
+
+}

Lts/Services/RecordingManagement/log4cxx_rec.props

@@ -0,0 +1,88 @@
+log4j.reset=true
+log4j.rootLogger=WARN, file
+
+# Service: RecordingManagement
+###########################################################
+
+#log4j.logger.REC=DEBUG
+#log4j.logger.REC.main=DEBUG
+#log4j.logger.REC.DdsHandlere=DEBUG
+#log4j.logger.REC.Settings=WARN
+#log4j.logger.REC.DdsSubscriber=INFO
+#log4j.logger.REC.MainHandler=DEBUG
+#log4j.logger.REC.HandlerRecordingData=DEBUG
+#log4j.logger.REC.HandlerRecTopic=DEBUG
+#log4j.logger.REC.ObjectRecTopic=DEBUG
+#log4j.logger.REC.SubscriberListener=DEBUG
+#log4j.logger.REC.DataReaderListener=DEBUG
+#log4j.logger.REC.DbWriterSnapshot=DEBUG
+
+# File
+###########################################################
+log4j.appender.file=Lts::RollingFileAppender
+log4j.appender.file.MaxFileSize=1MB
+log4j.appender.file.MaxBackupIndex=9
+log4j.appender.file.layout=org.apache.log4j.PatternLayout
+log4j.appender.file.layout.ConversionPattern=%d %-5p %c |#| %m (%.50F:%L) %n
+
+
+# Appnd_Console
+###########################################################
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern= %d %p %c %c{1} - %m (%-.30F:%L) %n
+
+
+# Appnd_Syslog
+###########################################################
+log4j.appender.syslog=org.apache.log4j.SyslogAppender
+log4j.appender.syslog.Threshold=ERROR
+log4j.appender.syslog.SysLogHost=localhost
+log4j.appender.syslog.Facility=LOCAL7
+log4j.appender.syslog.layout=org.apache.log4j.PatternLayout
+log4j.appender.syslog.layout.ConversionPattern= %-4r %-5p %c %x - %m
+log4j.appender.syslog.LevelRangeFilter.AcceptOnMatch=true
+log4j.appender.syslog.LevelRangeFilter.LevelMin=INFO
+log4j.appender.syslog.LevelRangeFilter.LevelMax=FATAL
+
+
+###########################################################
+###########################################################
+###########################################################
+# 1) Available Levels in LOG4CXX Version 0.10.0:
+# TRACE,
+# DEBUG,
+# INFO,
+# WARN,
+# ERROR
+# FATAL
+#
+# d = date/time
+# p = priority (Level)
+# c = category/channel
+# m = message
+# F = file
+# L = line
+# n = platform dependent line separator character or characters.
+# ....
+#
+# Examples for console:
+# a)
+# %d %p %c #%m# (%.50F:%L)%n
+# date/time
+#    priority
+#       category/channel
+#          #message#
+#                (Maximal 50Character des Filename von rechts:Line)
+#                       line separator
+#
+# b)
+# %d (%.15F:%L) %-5p %c |#| %m%n
+# date/time
+#    (Maximal 15Character des Filename von rechts:Line)
+#               priority, Right pad with spaces if the priority name is less than 5 characters long.
+#                    category/channel
+#                           message
+#                             line separator
+#
+# Examples for files:

Lts/Services/RecordingManagement/logger.cpp

@@ -0,0 +1,35 @@
+/**
+  * @RDE_HEADER@
+  * Building Config - logging facility
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#include "logger.h"
+#include <NGLLogger/NGLLoggerBootstrap.h>
+
+namespace RecordingManagement
+{
+
+LOGGER_T getLogger(const char *name)
+{
+  static const char ns[] = "REC.";
+  std::string loggerName;
+  const size_t len = strlen(name);
+  loggerName.reserve(sizeof(ns) + len);
+  loggerName.append(ns, sizeof(ns) - 1);
+  loggerName.append(name, len);
+
+  try
+  {
+    return NGL::NGLLoggerBootstrap::getNGLRootLogger().getNGLLogger( loggerName );
+  }
+  catch ( const std::exception &ex )
+  {
+    fprintf(stderr, "Error retrieving logger: %s", ex.what());
+    abort();
+  }
+}
+
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/logger.h

@@ -0,0 +1,24 @@
+/**
+  * @RDE_HEADER@
+  * Logging for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#pragma once
+
+#include <NGLLogger/INGLLogger.h>
+#include "rec_export.h"
+
+namespace RecordingManagement
+{
+using LOGGER_T = NGL::INGLLogger &;
+
+REC_COMMON_EXPORT LOGGER_T getLogger(const char *name);
+inline LOGGER_T getLogger(const QString &loggerName)
+{
+  return getLogger(loggerName.toUtf8().constData());
+}
+} // namespace RecordingManagement
+

Lts/Services/RecordingManagement/main.cpp

@@ -0,0 +1,54 @@
+/**
+  * @RDE_HEADER@
+  * main (RecordingManagement)
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#include <nglapplication/nglcoreapplication.h>
+#include <LtsCommandLineParser/LtsCommandLineParser.h>
+
+#include "main_handler.h"
+#include "logger.h"
+#include "settings.h"
+
+
+int main(int argc, char **argv)
+{
+  NGLCoreApplication app(argc, argv);
+
+  LtsCommandLineParser::LtsCommandLineParser options;
+  if (!options.init())
+    return -1;
+  app.createLogger("REC", options.exerciseId());
+
+  if (options.exerciseId() <= 0)
+  {
+    options.showHelp();
+    ERROR_LOG(app.baseLogger()) << "No RecordingManagement for exercise <= 0. Use a valid exerciseId.";
+    return -2;
+  }
+  RecordingManagement::Settings::setIniFile(options.iniFile());
+
+  DEBUG_LOG(app.baseLogger()) << "DomainId: #" << options.domainId() << ", exerciseId: #" << options.exerciseId();
+
+  int iRet = -3;
+  {
+    std::unique_ptr<RecordingManagement::MainHandler> initRecMgmt(new RecordingManagement::MainHandler());
+
+    if (!initRecMgmt->init(options.domainId(), options.exerciseId()))
+      return iRet;
+
+    STARTUP_LOG(app.baseLogger());
+
+    iRet = app.exec();
+
+    SHUTDOWN_LOG(app.baseLogger());
+  }
+
+  return iRet;
+}
+
+
+

Lts/Services/RecordingManagement/main_handler.cpp

@@ -0,0 +1,51 @@
+/**
+  * MainHandler for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#include "main_handler.h"
+
+#include "settings.h"
+#include "logger.h"
+#include <DDS/DDSConnector/dds_connector.h>
+#include <DDS/Core/pacemaker.h>
+#include <ngldb/db_service_handler.h>
+
+namespace RecordingManagement
+{
+
+MainHandler::MainHandler()
+  : m_logger(getLogger("MainHandler"))
+{}
+
+MainHandler::~MainHandler()
+{ 
+  m_dbWriter = nullptr;
+  m_dbHandler = nullptr;
+  m_ddsHandler = nullptr;
+}
+
+
+bool MainHandler::init(int domainId, int exerciseId)
+{
+  // initialize database
+  m_dbHandler = std::make_shared<DbHandler>();
+  if (!m_dbHandler->init(exerciseId, Settings::settings()))
+    return false;
+
+  // initialize DDS
+  m_ddsHandler = std::make_shared<DdsHandler>(domainId, exerciseId);
+  if (!m_ddsHandler->init())
+    return false;
+
+  m_dbWriter = std::make_shared<DbWriter>(m_ddsHandler->dds());
+  if (!m_dbWriter->init(m_dbHandler->db()))
+    return false;
+
+  m_ddsHandler->pacemaker()->setNewState(Core::RunningState::Running);
+
+  return true;
+}
+
+}  // RecordingManagement

Lts/Services/RecordingManagement/main_handler.h

@@ -0,0 +1,40 @@
+/**
+  * MainHandler for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#pragma once
+
+#include <memory>
+#include "rec_db.h"
+#include "dds_handler.h"
+
+#include "db_writer.h"
+#include "logger.h"
+
+namespace NGLDB
+{
+template <typename DB> class DbServiceHandler;
+}
+
+namespace RecordingManagement
+{
+using DbHandler = NGLDB::DbServiceHandler<DatabaseInterface>;
+class MainHandler final
+{
+public:
+  MainHandler();
+  ~MainHandler();
+
+  bool init(int domainId, int exerciseId);
+
+private:
+
+  std::shared_ptr<DdsHandler> m_ddsHandler;
+  std::shared_ptr<DbHandler>  m_dbHandler;
+  std::shared_ptr<DbWriter>   m_dbWriter;
+
+  LOGGER_T m_logger;
+};
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/rec.hpp

@@ -0,0 +1,3 @@
+// needed for rec_json.cpp until rec_json.cpp is not build anymore
+#include <vector>
+

Lts/Services/RecordingManagement/rec_db_impl_custom.cpp

@@ -0,0 +1,30 @@
+/**
+  * @RDE_HEADER@
+  * Custom db functions (extends rec_db_impl.cpp/.h)
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#include "rec_db_impl_custom.h"
+#include <ngldb/nsqlquery.h>
+#include <ngldb/sql_query_janitor.h>
+
+namespace RecordingManagement
+{
+// custom procs, body must be implemented by user
+bool DatabaseInterfaceImplCustom::removeNewerEqual(const Database::RecSnapshot &data)
+{
+  if (!m_query)
+    return false;
+
+  SqlQueryJanitor   janitor(m_query);
+
+  const QString stmt(QStringLiteral("DELETE FROM rec_snapshot WHERE svc_name = '") + data.svcName +
+                     QStringLiteral("' AND topic_name = '") + data.topicName +
+                     QStringLiteral("' AND time_stamp >= ") + QString::number(data.timeStamp));
+
+  return m_query->exec(stmt);
+}
+
+}

Lts/Services/RecordingManagement/rec_db_impl_custom.h

@@ -0,0 +1,22 @@
+/**
+  * @RDE_HEADER@
+  * Custom db functions (extends rec_db_impl.cpp/.h)
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+#pragma once
+
+#include <RecordingManagement/rec_db_impl.h>
+
+namespace RecordingManagement
+{
+class DatabaseInterfaceImplCustom final : public DatabaseInterfaceImpl
+{
+  Q_OBJECT
+public:
+  using DatabaseInterfaceImpl::DatabaseInterfaceImpl;
+
+  bool removeNewerEqual(const Database::RecSnapshot &data) override final;
+};
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/rec_export.h

@@ -0,0 +1,52 @@
+/**
+  * Export definitions
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  * Created by Hanna E. Rathje, SLSE, 2016-02-03
+  */
+#pragma once
+
+#include <QtCore/qglobal.h>
+
+#ifndef REC_COMMON_EXPORT
+# ifdef BUILD_STATIC_LIBRARIES
+#  define REC_COMMON_EXPORT
+# else
+#  if defined(MAKE_SVCRECCOMMON_LIB)
+#   define REC_COMMON_EXPORT Q_DECL_EXPORT
+#   ifndef WIN32
+#     ifdef NDDSUSERDllExport
+#      undef NDDSUSERDllExport
+#     endif
+#     define NDDSUSERDllExport Q_DECL_EXPORT
+#   endif
+#  else
+#   define REC_COMMON_EXPORT Q_DECL_IMPORT
+#  endif
+# endif
+#endif
+
+#ifndef REC_CLIENT_EXPORT
+# ifdef BUILD_STATIC_LIBRARIES
+#  define REC_CLIENT_EXPORT
+# else
+#  if defined(MAKE_SVCRECCLIENT_LIB)
+#   define REC_CLIENT_EXPORT Q_DECL_EXPORT
+#  else
+#   define REC_CLIENT_EXPORT Q_DECL_IMPORT
+#  endif
+# endif
+#endif
+
+#ifndef REC_SERVER_EXPORT
+# ifdef BUILD_STATIC_LIBRARIES
+#  define REC_SERVER_EXPORT
+# else
+#  if defined(MAKE_SVCRECSERVER_LIB)
+#   define REC_SERVER_EXPORT Q_DECL_EXPORT
+#  else
+#   define REC_SERVER_EXPORT Q_DECL_IMPORT
+#  endif
+# endif
+#endif

Lts/Services/RecordingManagement/rec_service.xml

@@ -0,0 +1,77 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<service namespace="RecordingManagement" abbreviation="REC" procNames="RecordingManagement" exerciseDependency="Exercise" groups="ExercisePreparation,ExerciseExecution,ExercisePaused">
+  <rcs>
+    <Id Id="$Id$"/>
+    <HeadURL HeadURL="$HeadURL$"/>
+    <Author Author="$Author$"/>
+  </rcs>
+  <tables>
+    <table name="rec_recorded_topic" comment="All topics available for recording in an exercise and their recording state">
+      <column name="id" datatype="bigint" default="0" isPrimaryKey="1">
+        <sequence start="1" increment="1"/>
+      </column>
+      <column name="rec_topic_id" datatype="bigint" isForeignKey="1"/>
+      <column name="recording_type_id" datatype="integer" isForeignKey="1"/>
+      <column name="recorded" datatype="boolean" default="true"/>
+      <foreignKey name="FK_rec_topic_rec_recording_type">
+        <source table="rec_recorded_topic" column="recording_type_id"/>
+        <destination table="rec_recording_type" column="id"/>
+      </foreignKey>
+      <foreignKey name="FK_rec_recorded_topic_rec_topic">
+        <source table="rec_recorded_topic" column="rec_topic_id"/>
+        <destination table="rec_topic" column="id"/>
+      </foreignKey>
+    </table>
+    <table name="rec_recording_type">
+      <column name="id" datatype="integer" isPrimaryKey="1"/>
+      <column name="name" datatype="text"/>
+    </table>
+    <table name="rec_snapshot">
+      <column name="id" datatype="bigint" isPrimaryKey="1">
+        <sequence start="1" increment="1"/>
+      </column>
+      <column name="time_stamp" datatype="bigint"/>
+      <column name="topic_name" datatype="text"/>
+      <column name="svc_name" datatype="text"/>
+      <column name="rec_id_array" datatype="bigint[][]"/>
+      <constraint name="IDX_time_stamp" type="index">
+        <column name="time_stamp"/>
+      </constraint>
+      <constraint name="IDX_svc_topic_name" type="index">
+        <column name="svc_name"/>
+        <column name="topic_name"/>
+      </constraint>
+    </table>
+    <table name="rec_topic">
+      <column name="id" datatype="bigint" isPrimaryKey="1">
+        <sequence start="1" increment="1"/>
+      </column>
+      <column name="name" datatype="text"/>
+    </table>
+  </tables>
+  <views/>
+  <dbServiceStructs>
+    <serviceStruct name="RecSnapshot" table="rec_snapshot"/>
+    <serviceStruct name="RecTopic" table="rec_recorded_topic">
+      <function name="removeNewerEqual" returns="bool">
+        <parameter pos="1" name="data" type="const Database::RecSnapshot &amp;"/>
+      </function>
+    </serviceStruct>
+    <table name="rec_snapshot"/>
+    <table name="rec_recorded_topic" base="rec_topic"/>
+    <table name="rec_topic"/>
+  </dbServiceStructs>
+  <topicTypes/>
+  <functionTopics/>
+  <enumerations>
+    <enumeration name="RecordingType" table="rec_recording_type">
+      <attribute key="UnknownData" value="0"/>
+      <attribute key="SituationData" value="1"/>
+      <attribute key="ReportData" value="2"/>
+      <attribute key="EventData" value="3"/>
+      <attribute key="OtherData" value="4"/>
+      <attribute key="All" value="5"/>
+    </enumeration>
+  </enumerations>
+  <operatorRights offset="30"/>
+</service>

Lts/Services/RecordingManagement/settings.cpp

@@ -0,0 +1,47 @@
+/**
+  * Settings for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#include "settings.h"
+
+#include <QtCore/QSettings>
+#include <nglsettings/nglsettings_settings.h>
+#include <ngltools/ngltools_filesystem.h>
+
+namespace RecordingManagement
+{
+class NGLSettingsClass : public NGL::Settings
+{
+public:
+  NGLSettingsClass()
+    : NGL::Settings(s_iniFile.isEmpty() ? NGL::findProjectConfigFile() : s_iniFile)
+  {}
+  static void setIniFile(const QString &fn)
+  {
+    s_iniFile = fn;
+  }
+  static QString s_iniFile;
+};
+QString NGLSettingsClass::s_iniFile;
+Q_GLOBAL_STATIC(NGLSettingsClass, s)
+
+//-----------------------------------------------------------------------------
+NGL::Settings *Settings::settings()
+{
+  return s();
+}
+//-----------------------------------------------------------------------------
+void Settings::setIniFile(const QString &ini)
+{
+  NGLSettingsClass::setIniFile(ini);
+}
+
+qint64 Settings::updateFrequency()
+{
+  return settings()->settings()->value(QStringLiteral("RecordingManagement/UpdateFrequency"), 1000).toLongLong();  // msecs
+}
+
+} // namespace RecordingManagement

Lts/Services/RecordingManagement/settings.h

@@ -0,0 +1,27 @@
+/**
+  * Settings for RecordingManagement
+  * $Id$
+  * $HeadURL$
+  * $Author$
+  */
+
+#pragma once
+#include <QString>
+
+namespace NGL
+{
+class Settings;
+}
+
+namespace RecordingManagement
+{
+namespace Settings
+{
+//!< The real NGL::Settings - object
+NGL::Settings *settings();
+void setIniFile(const QString &ini);
+qint64 updateFrequency();
+
+}
+
+} // namespace RecordingManagement