qt5-legatus-legacy/Lts/Services/ServiceManagement/service_handler.cpp

/**
  * The ServiceHandler class handles all service related classes and writes the services state to DDS.
  * $Id$
  * $HeadURL$
  * $Author$
  * Created by Hannes Baeuerle, SLSE, 2016-02-15
  */

#include "service_handler.h"

#include <QCoreApplication>
#include <QDateTime>
#include <QTimer>

#include "service.h"
#include "service_process.h"
#include "dds_handler.h"
#include "settings.h"
#include <LtsUtilsCore/data_modification_enum.h>

ServiceManagement::ServiceHandler::ServiceHandler(const std::shared_ptr<DdsHandler> &ddsHandler,
                                                  const std::shared_ptr<LicenceDonator> &licenceDonator)
  : ServiceDonator()
  , Interface::Fts::RestartServiceRequestHandler(ddsHandler->fts())
  , m_heartBeatObserverTimer(new QTimer(this))
  , m_ddsHandler(ddsHandler)
  , m_licenceDonator(licenceDonator)
  , m_logger(getLogger("ServiceHandler"))
{
  connect(qApp, &QCoreApplication::aboutToQuit,
          this, &ServiceHandler::stopAllServices);
}

ServiceManagement::ServiceHandler::~ServiceHandler()
{
  qDeleteAll(m_services);
  m_services.clear();
}

bool ServiceManagement::ServiceHandler::init()
{
  DEBUG_LOG(m_logger) << "Initializing ServiceHandler ...";

  connect(m_heartBeatObserverTimer, &QTimer::timeout,
          this, &ServiceHandler::onHeartBeatObserverTimerTimeout);


  HEARTBEAT_OBSERVER_INTERVAL = Settings::observerIntervall();
  DEBUG_LOG(m_logger) << HEARTBEAT_OBSERVER_INTERVAL;
  m_heartBeatObserverTimer->start(HEARTBEAT_OBSERVER_INTERVAL);

  DEBUG_LOG(m_logger) << "ServiceHandler successfully initialized.";
  return true;
}

void ServiceManagement::ServiceHandler::addService(ServiceManagement::Service *service)
{
  service->setParent(this);
  service->setLicenceDonator(m_licenceDonator);

  connect(service, &Service::newServiceInfo,
          this, &ServiceHandler::onNewServiceInfoFromProc);

  m_services.append(service);
}

ServiceManagement::ServiceProcess *ServiceManagement::ServiceHandler::getProcess(int serviceId,
                                                                                 const GroupRequest &groupData)
{
  Service *service = serviceFromId(serviceId);

  if (service == nullptr)
    return nullptr;

  return service->toServiceProcess(groupData);
}

void ServiceManagement::ServiceHandler::onRestartServiceRequest(const rti::core::SampleIdentity &cookie,
                                                                const RestartServiceRequest &request)
{
  DEBUG_LOG(m_logger) << "onRestartServiceRequest " << request;

  ServiceProcess *proc = serviceProcessFromServiceProcId(request.id());

  Core::FunctionReplyObject ret(request.id());
  if (proc == nullptr)
  {
    INFO_LOG(m_logger) << "Got restart request for non existing service ID "
                       << request.id();
    ret.setStr(tr_lts("dds", "There is no ServiceProcess with id %1")).arg(request.id());
    m_ddsHandler->fts()->restartServiceReply(cookie, ret);
    return;
  }

  DEBUG_LOG(m_logger) << "Restart request for service "
                      << proc->serviceName()
                      << ", domain "
                      << proc->domainId()
                      << ", exercisePartitionId "
                      << proc->exercisePartitionId()
                      << ", replayPartitionId "
                      << proc->replayPartitionId();

  proc->restart();

  m_ddsHandler->fts()->restartServiceReply(cookie, ret);
}

void ServiceManagement::ServiceHandler::stopAllServices()
{
  INFO_LOG(m_logger) << "Stopping all Services ...";

  // shutdown async
  for (Service *service : qAsConst(m_services))
    service->stopAllProcs();
  for (Service *service : qAsConst(m_services))
    service->waitForStopped();

  INFO_LOG(m_logger) << "... all Services stopped :)";
}

void ServiceManagement::ServiceHandler::onNewServiceInfoFromProc(const ServiceInformation &info)
{
  TRACE_LOG(m_logger) << "Publishing new ServiceInformation: " << info;

  m_ddsHandler->dts()->publish(info);

  m_unknownPidsWarned.remove(info.pid());
}

void ServiceManagement::ServiceHandler::onNewServiceInfoFromDds(const Core::CurrentRunningState &data, DataModification type)
{
  if (type == DataModification::eRemoved)
    return;

  TRACE_LOG(m_logger) << "Got new CurrentRunningState: " << data;

  if (!data.pid())
  {
    INFO_LOG(m_logger) << "PID 0 is not allowed.";
    return;
  }

  ServiceProcess *proc = serviceProcessFromPid(data.pid());
  Core::CurrentRunningState retData = data;

  if (proc == nullptr)
  {
    bool found = false;
    for (Service *currentService : qAsConst(m_services))
    {
      for (ServiceProcess *currentProc : qAsConst(currentService->allProcs()))
      {
        if (currentProc->remotePid() == data.pid())
        {
          DEBUG_LOG(m_logger) << "Found my prog " << data.pid() << " to "<< currentProc->pid() << " " << currentProc->serviceName();
          proc = currentProc;
          retData.pid() = proc->processId();
          found = true;
          break;
        }
      }
      if (found)
        break;
    }
  }


  if (proc == nullptr)
  {
    if (!m_unknownPidsWarned.contains(data.pid()))
    {
      ERROR_LOG(m_logger) << "Got CurrentRunningState for non existing process with PID "
                          << data.pid();
      m_unknownPidsWarned += data.pid();
    }
    return;
  }

  proc->setNewStateFromHeartbeat(retData);
}

ServiceManagement::Service *ServiceManagement::ServiceHandler::serviceFromId(int id)
{
  for (Service *service : qAsConst(m_services))
  {
    if (service->id() == id)
      return service;
  }
  return nullptr;
}

ServiceManagement::ServiceProcess *ServiceManagement::ServiceHandler::serviceProcessFromPid(qint64 pid)
{
  for (Service *service : qAsConst(m_services))
  {
    ServiceProcess *result = service->procFromPid(pid);
    if (result)
      return result;
  }
  return nullptr;
}

ServiceManagement::ServiceProcess *ServiceManagement::ServiceHandler::serviceProcessFromServiceProcId(int32_t id)
{
  for (Service *service : qAsConst(m_services))
  {
    ServiceProcess *result = service->procFromServiceProcId(id);
    if (result)
      return result;
  }
  return nullptr;
}

void ServiceManagement::ServiceHandler::onHeartBeatObserverTimerTimeout()
{
  const qint64 tNow = QDateTime::currentMSecsSinceEpoch();
  const qint64 minHeartbeatAge = tNow - HEARTBEAT_OBSERVER_INTERVAL;
  const qint64 minHeartbeatAgeStartup = tNow - HEARTBEAT_OBSERVER_INTERVAL * 6;

  for (Service *currentService : qAsConst(m_services))
  {
    for (ServiceProcess *currentProc : qAsConst(currentService->allProcs()))
    {
      // dev mode for this proc -> we're not repsonsible for
      if (currentProc->devMode())
        continue;
      // stopped on operator demand -> we're not responsible for
      if (currentProc->stoppedFromRequest())
        continue;
      // there is something terribly wrong -> ignore
      if (currentProc->failedStartingAttempts() > 3)
        continue;

      const qint64 minAge = (currentProc->runningState() == Core::RunningState_def::Starting)
          ? minHeartbeatAgeStartup
          : minHeartbeatAge;

      if (currentProc->lastHeartbeatMSec() < minAge)
      {
        WARNING_LOG(m_logger) << "Heartbeat TIMEOUT for " << currentProc->debugInfo()
                              << " - Restarting ..."
                              << " min heartbeat age: " << QDateTime::fromMSecsSinceEpoch(minHeartbeatAge).toString(QStringLiteral("HH:mm:ss"));
        currentProc->addStartAttempt();
        currentProc->restart();
      }
    }
  }
}