qt5-legatus-legacy/Lts/Services/ServiceManagement/service_process.cpp

/**
  * The ServiceProcess class represents a service which was startet as a process
  * $Id$
  * $HeadURL$
  * $Author$
  * Created by Hannes Baeuerle, SLSE, 2016-02-15
  */

#include "service_process.h"
#include "service.h"
#include "licence_donator.h"
#include "settings.h"
#include <ServiceManagement/scm_dts_types.h>

#include <QtCore/QDateTime>
#include <QtCore/QTimer>

int ServiceManagement::ServiceProcess::s_serviceProcessIdCounter = 0;

ServiceManagement::ServiceProcess::ServiceProcess(const GroupRequest &groupData, ServiceManagement::Service *parentService)
  : QProcess(parentService)
  , m_serviceId(parentService->id())
  , m_serviceProcessId(s_serviceProcessIdCounter++)
  , m_abbreviation(parentService->abbreviation())
  , m_name(parentService->name())
  , m_prio(parentService->prio())
  , m_groupData(groupData)
  , m_remotePid(-1)
  , m_runningState(Core::RunningState::NotYetStarted)
  , m_lastHeartbeatMSec(0)
  , m_parentService(parentService)
  , m_hasLicence(false)
  , m_isInDevMode(parentService->devMode())
  , m_failedStartAttemptCounter(0)
  , m_stoppedFromRequest(false)
  , m_pidProc(new QProcess(this))
  , m_logger(getLogger("ServiceProcess"))
{
  QStringList args({QStringLiteral("-e"), QString::number(m_groupData.exercisePartitionId()),
                    QStringLiteral("-d"), QString::number(m_groupData.domainId())});

  bool        isInReplayGroup = false;
  for(const QString &currentGroup : parentService->groups())
  {
    if (currentGroup.startsWith(QLatin1String("Replay")) || currentGroup.startsWith(QLatin1String("Playback")))
    {
      isInReplayGroup = true;
      break;
    }
  }

  if (isInReplayGroup)
    args << QStringLiteral("-r") << QString::number(m_groupData.replayPartitionId());
  args.reserve(args.size() + m_groupData.additionalStartParameter().size());
  for (const std::string &arg : m_groupData.additionalStartParameter())
    args << QString::fromStdString(arg);

  setArguments(args);

#ifndef LTS_DEBUG
  if (m_name == QStringLiteral("HighLevelArchitecture"))
  {
    DEBUG_LOG(m_logger) <<" START HLA REMOTE";
    killallRemote();

    //QString sshCmd =  QStringLiteral("ssh ctcap03 '/shd/CTC/SV/bin/svcHighLevelArchitecture %1 &'").arg(arguments().join(QLatin1Char(' ')));
    QStringList remoteCommand;
    remoteCommand.append(Settings::hlaRemoteServer());
    remoteCommand.append(QStringLiteral("/shd/CTC/SV/bin/svcHighLevelArchitecture "));
    remoteCommand.append(arguments());
    setArguments(remoteCommand);
    setProgram(QStringLiteral("ssh"));
  }

  if (m_name == QStringLiteral("Buildings"))
  {
    DEBUG_LOG(m_logger) <<" START Buildings Remote";
    killallRemote();

    //QString sshCmd =  QStringLiteral("ssh ctcap03 '/shd/CTC/SV/bin/svcHighLevelArchitecture %1 &'").arg(arguments().join(QLatin1Char(' ')));
    QStringList remoteCommand;
    remoteCommand.append(Settings::buildingsRemoteServer());
    remoteCommand.append(QStringLiteral("/shd/CTC/SV/bin/svcBuildings "));
    remoteCommand.append(arguments());
    setArguments(remoteCommand);
    setProgram(QStringLiteral("ssh"));
  }
#endif


  connect(this, &ServiceProcess::stateChanged,
          this, &ServiceProcess::onNewInfoFromProc);
}

ServiceManagement::ServiceProcess::~ServiceProcess()
{
  disconnect(this);
#ifndef LTS_DEBUG
  if (m_name == QStringLiteral("HighLevelArchitecture"))
  {
    killallRemote();
  }
#endif
  if (state() == QProcess::Running)
  {
    terminate();
    waitForFinished(5000);
    if (state() == QProcess::Running)
      kill();
  }
}

Core::RunningState ServiceManagement::ServiceProcess::runningState() const
{
  return m_runningState;
}

bool ServiceManagement::ServiceProcess::start(OpenMode mode)
{
  DEBUG_LOG(m_logger) << "Starting ServiceProcess "
                      << debugInfo()
                      << " executable: " << program()
                      << " in dir: " << workingDirectory()
                      << " with arguments " << arguments().join(QLatin1Char(' '))
                      << " prio=" << m_prio;
  if (m_parentService->licenceDonator()->getLicence(m_name))
  {
    m_hasLicence = true;
    m_stoppedFromRequest = false;
#ifndef LTS_DEBUG
    if (m_name == QStringLiteral("HighLevelArchitecture"))
    {
      DEBUG_LOG(m_logger) <<" START HLA REMOTE";

      //QString sshCmd =  QStringLiteral("ssh ctcap03 '/shd/CTC/SV/bin/svcHighLevelArchitecture %1 &echo $! '").arg(arguments().join(QLatin1Char(' ')));
      //DEBUG_LOG(m_logger) << sshCmd;
      setProgram(QStringLiteral("ssh"));

      if (state() == QProcess::NotRunning)
      {
        QProcess::start(mode);
        waitForStarted(5000);
      }
      DEBUG_LOG(m_logger) << program() << " " << arguments().join(QLatin1Char(' '));

      m_pidProc->kill();
      m_pidProc->setProgram(QStringLiteral("ssh"));
      //DEBUG_LOG(m_logger) << sshCmd;
      QStringList args({QStringLiteral("-e"), QString::number(m_groupData.exercisePartitionId()),
                        QStringLiteral("-d"), QString::number(m_groupData.domainId())});
      setProgram(QStringLiteral("ssh"));
      QStringList remoteCommand2;
      remoteCommand2.append(Settings::hlaRemoteServer());
      remoteCommand2.append(QStringLiteral("pgrep -f \'svcHighLevelArchitecture %1\'").arg(args.join(QLatin1Char(' '))));
      m_pidProc->setArguments(remoteCommand2);
      //args.append(QStringLiteral("pkill -9 -f \"svcHighLevelArchitecture %1\"").arg(arg.join(QLatin1Char(' '))));


      m_pidProc->setProcessChannelMode(QProcess::MergedChannels);
      m_pidProc->disconnect();
      connect(m_pidProc, &QProcess::readyRead, this , &ServiceProcess::onDataFromPidProc, Qt::UniqueConnection);

      QTimer::singleShot(3000, this, [this](){
        m_pidProc->start();
        m_pidProc->waitForReadyRead(5000);
      });

      QTimer::singleShot(15000, this, [this](){
        m_pidProc->start();
        m_pidProc->waitForReadyRead(5000);
      });

      DEBUG_LOG(m_logger) << m_pidProc->program() << " " << m_pidProc->arguments().join(QLatin1Char(' '));

    }else if (m_name == QStringLiteral("Buildings"))
    {
      DEBUG_LOG(m_logger) <<" START Buildings REMOTE";
      setProgram(QStringLiteral("ssh"));
      //QString sshCmd =  QStringLiteral("ssh ctcap03 '/shd/CTC/SV/bin/svcHighLevelArchitecture %1 &echo $! '").arg(arguments().join(QLatin1Char(' ')));
      //DEBUG_LOG(m_logger) << sshCmd;

      if (state() == QProcess::NotRunning)
      {
        QProcess::start(mode);
        waitForStarted(5000);
      }
      DEBUG_LOG(m_logger) << program() << " " << arguments().join(QLatin1Char(' '));


      m_pidProc->kill();
      m_pidProc->setProgram(QStringLiteral("ssh"));
      //DEBUG_LOG(m_logger) << sshCmd;
      QStringList args({QStringLiteral("-e"), QString::number(m_groupData.exercisePartitionId()),
                        QStringLiteral("-d"), QString::number(m_groupData.domainId())});
      setProgram(QStringLiteral("ssh"));
      QStringList remoteCommand2;
      remoteCommand2.append(Settings::hlaRemoteServer());
      remoteCommand2.append(QStringLiteral("pgrep -f \'svcBuildings %1\'").arg(args.join(QLatin1Char(' '))));
      m_pidProc->setArguments(remoteCommand2);
      //args.append(QStringLiteral("pkill -9 -f \"svcHighLevelArchitecture %1\"").arg(arg.join(QLatin1Char(' '))));


      m_pidProc->setProcessChannelMode(QProcess::MergedChannels);
      m_pidProc->disconnect();

      connect(m_pidProc, &QProcess::readyRead, this , &ServiceProcess::onDataFromPidProc, Qt::UniqueConnection);

      QTimer::singleShot(9000, this, [this](){
        m_pidProc->start();
        m_pidProc->waitForReadyRead(5000);
      });


      QTimer::singleShot(15000, this, [this](){
        m_pidProc->start();
        m_pidProc->waitForReadyRead(5000);
      });

      DEBUG_LOG(m_logger) << m_pidProc->program() << " " << m_pidProc->arguments().join(QLatin1Char(' '));
    }
    else
#endif
    {
      if (state() == QProcess::NotRunning)
        QProcess::start(mode);
    }


    m_lastHeartbeatMSec = QDateTime::currentMSecsSinceEpoch();
    return true;
  }
  else
  {
    ERROR_LOG(m_logger) << "Could not start service "
                        << debugInfo()
                        << " because there is no licence for it.";
    return false;
  }
}

int ServiceManagement::ServiceProcess::serviceProcessId() const
{
  return m_serviceProcessId;
}

void ServiceManagement::ServiceProcess::setNewStateFromHeartbeat(const Core::CurrentRunningState &data)
{
  DEBUG_LOG(m_logger) << "Heartbeat for " << debugInfo() << ", running state: " << data.runningState();
  m_failedStartAttemptCounter = 0;
  m_lastHeartbeatMSec = QDateTime::currentMSecsSinceEpoch();
  m_runningState = data.runningState();
  m_ports = data.ports();

  std::string es;
  if (data.errorString().is_set())
    es = data.errorString().get();

  ServiceInformation info(m_serviceProcessId,
                          m_lastHeartbeatMSec,
                          m_name.toStdString(),
                          m_runningState,
                          es,
                          usablePid());

  emit newServiceInformation(info);
}

void ServiceManagement::ServiceProcess::killallRemote()
{
  QStringList arg({QStringLiteral("-e"), QString::number(m_groupData.exercisePartitionId()),
                   QStringLiteral("-d"), QString::number(m_groupData.domainId())});
  QProcess sh;
  sh.setProgram(QStringLiteral("ssh"));
  QStringList args;
  // QString killForm = QLatin1String("");
  QString killForm = QStringLiteral("-15");
  if (m_name == QStringLiteral("Buildings"))
    args.append(Settings::buildingsRemoteServer());
  else
  {
    args.append(Settings::hlaRemoteServer());
    killForm = QStringLiteral("-9");
  }

  args.append(QStringLiteral("pkill %1 -f \"%2 %3\"").arg(killForm, m_name,arg.join(QLatin1Char(' '))));
  sh.setArguments(args);
#if QT_VERSION >= QT_VERSION_CHECK(5, 10, 0)
  sh.startDetached();
#else
  sh.start();
#endif  DEBUG_LOG(m_logger) << "kill remote " << sh.program() << " " << args.join(QLatin1Char(' '));
}

void ServiceManagement::ServiceProcess::restart()
{
#ifndef LTS_DEBUG

  if (m_name == QStringLiteral("HighLevelArchitecture") || m_name == QStringLiteral("Buildings"))
  {
    killallRemote();
  }
#endif
  // else
  terminate();  // already trigger terminate here to shut down apps in parallel
  QTimer::singleShot(0, this, &ServiceProcess::doRestart);
}

void ServiceManagement::ServiceProcess::doRestart()
{
  if (m_hasLicence)
  {
    m_hasLicence = false;
    m_parentService->licenceDonator()->freeLicence(m_name);
  }
#ifndef LTS_DEBUG

  if (m_name == QStringLiteral("HighLevelArchitecture"))
  {
    //remoteKill
    killallRemote();
  }
#endif
  // else
  {
    terminate();
    waitForFinished(5000);
    if (state() == QProcess::Running)
      kill();

  }

  QTimer::singleShot(100, this, [this]{ start(); });
}

void ServiceManagement::ServiceProcess::onDataFromPidProc()
{
  const QByteArray &ret = m_pidProc->readAll();
  FATAL_LOG(m_logger) << "New Data  Id from remote" << ret;
  int newId = ret.trimmed().toInt();
  if (newId > 0)
  {
    m_remotePid = newId;
    FATAL_LOG(m_logger) << "New QProcess Id from remote" << m_name << "   pid= "<< m_remotePid ;
    m_pidProc->kill();
  }

}

void ServiceManagement::ServiceProcess::stopFromRequest()
{
  m_stoppedFromRequest = true;
  QProcess::terminate();
}

unsigned short ServiceManagement::ServiceProcess::failedStartingAttempts() const
{
  return m_failedStartAttemptCounter;
}

void ServiceManagement::ServiceProcess::addStartAttempt()
{
  ++m_failedStartAttemptCounter;
}

qint64 ServiceManagement::ServiceProcess::lastHeartbeatMSec() const
{
  return m_lastHeartbeatMSec;
}

void ServiceManagement::ServiceProcess::setLastHeartbeatMSec(qint64 lastHeartbeatMSec)
{
  m_lastHeartbeatMSec = lastHeartbeatMSec;
}

QString ServiceManagement::ServiceProcess::serviceName() const
{
  return m_name;
}

QString ServiceManagement::ServiceProcess::abbreviation() const
{
  return m_abbreviation;
}

dds::core::optional<std::vector<Core::PortInformation> > ServiceManagement::ServiceProcess::ports() const
{
  return m_ports;
}

bool ServiceManagement::ServiceProcess::devMode() const
{
  return m_isInDevMode;
}

bool ServiceManagement::ServiceProcess::stoppedFromRequest() const
{
  return m_stoppedFromRequest;
}

void ServiceManagement::ServiceProcess::setNewStateFromChangeRequest(const Core::RunningState &newState)
{
  if (m_runningState != newState)
  {
    switch (newState.underlying())
    {
    case Core::RunningState::Running:
    case Core::RunningState::Starting:
      if (state() != QProcess::Running)
        start();
      break;
    case Core::RunningState::StoppedNormally:
    case Core::RunningState::Stopping:
      terminate();
      waitForFinished(5000);
      break;
    default:
      return;
    }
  }
}

void ServiceManagement::ServiceProcess::onNewInfoFromProc(ProcessState state)
{
  DEBUG_LOG(m_logger) << "New state for " << debugInfo() << ": " << state;
  QString errString;
  switch (state)
  {
  case QProcess::NotRunning:
    if (exitStatus() == QProcess::NormalExit)
    {
      if (exitCode() == 0)
        DEBUG_LOG(m_logger) << "Normal exit for " << debugInfo()
                            << ": " << readAllStandardError() << ", " << readAllStandardOutput();
      else
        ERROR_LOG(m_logger) << "Normal exit with exit code " << exitCode() << " for " << debugInfo()
                            << ": " << readAllStandardError() << ", " << readAllStandardOutput();
      m_runningState = Core::RunningState::StoppedNormally;
    }
    else
    {
      switch (error())
      {
      case QProcess::FailedToStart:
        errString = QLatin1String("Got QProcess::FailedToStart error from Service process with exit code ") + QString::number(exitCode());
        m_runningState = Core::RunningState::StartingError;
        break;
      case QProcess::Crashed:
        errString = QLatin1String("Got QProcess::Crashed error from Service Process with exit code ") + QString::number(exitCode());
        m_runningState = Core::RunningState::StoppedError;
        break;
      default:
        errString = QLatin1String("Got unkown error.");
        m_runningState = Core::RunningState::StoppedError;
      }
      errString += QLatin1String(" error string: ") + errorString();
      ERROR_LOG(m_logger) << "Crashed Error for " << debugInfo() << ", error: " << errorString() << ":\n"
                          << readAll();
    }
    m_parentService->licenceDonator()->freeLicence(m_name);
    m_hasLicence = false;
    m_lastHeartbeatMSec = 0;
    break;
  case QProcess::Starting:
    m_runningState = Core::RunningState::Starting;
    break;
  case QProcess::Running:
    DEBUG_LOG(m_logger) << "Process " << m_name
                        << " has now the PID " << usablePid();
  }

  ServiceInformation info(m_serviceProcessId,
                          QDateTime::currentMSecsSinceEpoch(),
                          m_name.toStdString(),
                          m_runningState,
                          errString.toStdString(),
                          usablePid());

  emit newServiceInformation(info);
}

QString ServiceManagement::ServiceProcess::debugInfo() const
{
  QString partitionStr;
  if (replayPartitionId() > 0)
    partitionStr = QStringLiteral("%1 (replay)").arg(replayPartitionId());
  else
    partitionStr = QString::number(exercisePartitionId());
  return QStringLiteral("%1, domain: %2, partition: %3, PID: %4, last heartbeat: %5, arguments: '%6' this: 0x%7")
      .arg(serviceName())
      .arg(domainId())
      .arg(partitionStr)
      .arg(usablePid())
      .arg(QDateTime::fromMSecsSinceEpoch(m_lastHeartbeatMSec).toString(QStringLiteral("HH:mm:ss")),
           arguments().join(QLatin1Char(' ')))
      .arg(reinterpret_cast<qint64>(this), 8, 16, QLatin1Char('0'));
}