/** * WebSocket connection to AVS to retrieve AVS events * $Id$ * $HeadURL$ * $Author$ * Created by Christian Ehrlicher, SLSE, 2017-02-06 */ #include "web_socket_avs.h" #include "web_socket_server.h" #include "main_handler.h" #include "settings.h" #include #include #include #include #include #include ProjectDataManagement::EventType avsEventTypeToPdmEventType(int avsEventType) { ProjectDataManagement::EventType eventType; switch(avsEventType) { case 305: //MVS case 301: // CAMERA_AUDIO case 302: // CAMERA_VIDEO eventType = ProjectDataManagement::EventType::CameraVideo; break; case 303: // RADIO_CNR eventType = ProjectDataManagement::EventType::RadioCnr; break; case 304: // RADIO_TVC eventType = ProjectDataManagement::EventType::RadioTvc; break; default: eventType = ProjectDataManagement::EventType::Unknown; break; } return eventType; } namespace ExerciseEventManagement { WebSocketAvs::WebSocketAvs(MainHandler *mainHandler, const std::shared_ptr &ddsHandler, const std::shared_ptr &db, const std::shared_ptr &eventServer, const std::shared_ptr &avmDtc, const std::shared_ptr &ismDtc, int exerciseId, QObject *parent) : QObject(parent) , m_ddsHandler(ddsHandler) , m_dbHandler(db) , m_eventServer(eventServer) , m_avmDtc(avmDtc) , m_ismDtc(ismDtc) , m_mainHandler(mainHandler) , m_wsSocket(nullptr) , m_wsReconnectTimer(new QTimer(this)) , m_cleanupTimer(new QTimer(this)) , m_exerciseId(exerciseId) , m_lastUpdate(0) , m_bWebSocketDisconnectedWarnedOnce(false) , m_logger(getLogger("WebSocketAvs")) { connect(m_wsReconnectTimer, &QTimer::timeout, this, &WebSocketAvs::openWebSocket); connect(m_cleanupTimer, &QTimer::timeout, this, &WebSocketAvs::onCleanUpDbInternal); //if there is a positive time start it //if not: don't do it! int cleanupTimeout = Settings::getCleanupTimeout(); INFO_LOG(m_logger) << "cleanup Timer Interval: " << cleanupTimeout; if (cleanupTimeout > 0) { m_cleanupTimer->setInterval(cleanupTimeout); m_cleanupTimer->start(); INFO_LOG(m_logger) << "Started cleanup Timer with Interval: " << cleanupTimeout; } } WebSocketAvs::~WebSocketAvs() { } void WebSocketAvs::init() { DEBUG_LOG(m_logger) << "WebSocketAvs::init"; m_wsReconnectTimer->start(1000); //onCleanUpDbInternal(); } void WebSocketAvs::setWsUrl(const QString &url) { DEBUG_LOG(m_logger) << "Url: " << url; m_wsUrl = url; } AvsData WebSocketAvs::jsonObjectToAvsData(const QJsonObject &obj) { AvsData data; data.id = 0; data.eventTime = obj[QStringLiteral("eventTime")].toDouble(); data.endTime = obj[QStringLiteral("endTime")].toDouble(); data.streamName = obj[QStringLiteral("streamName")].toString(); int avsEventType = obj[QStringLiteral("eventType")].toInt(); ProjectDataManagement::EventType eventType; switch(avsEventType) { case 301: // CAMERA_AUDIO case 302: // CAMERA_VIDEO eventType = ProjectDataManagement::EventType::CameraVideo; break; case 303: // RADIO_CNR eventType = ProjectDataManagement::EventType::RadioCnr; break; case 304: // RADIO_TVC eventType = ProjectDataManagement::EventType::RadioTvc; break; default: eventType = ProjectDataManagement::EventType::Unknown; ERROR_LOG(m_logger) << "Unknown avsEventType: " << avsEventType; break; } data.eventType = eventType.underlying(); int mountingPointId = 0; if (m_avmDtc->mountingPointForStreamName(data.streamName, mountingPointId)) { int mainStructureElementId = 0; if (m_ismDtc->getBuildingId(mountingPointId, mainStructureElementId)) data.mainStructureElementId = mainStructureElementId; } return data; } Database::EemEvent WebSocketAvs::avsDataToEemEvent(const AvsData &data) { Database::EemEvent eemEvent; eemEvent.id = data.id; eemEvent.eventTypeId = data.eventType.underlying(); eemEvent.sampleTime = data.eventTime; QJsonObject obj; obj[QStringLiteral("streamName")] = data.streamName; if (data.endTime > 0) { qint64 duration = data.endTime - data.eventTime; obj[QStringLiteral("duration")] = duration; } obj[QStringLiteral("endTime")] = data.endTime; obj[QStringLiteral("MainStructureElementId")] = data.mainStructureElementId; eemEvent.properties = QString::fromUtf8(QJsonDocument(obj).toJson()); return eemEvent; } ExerciseEventManagement::Event WebSocketAvs::avsDataToEvent(const AvsData &data) { ExerciseEventManagement::Event event; event.id(data.id); event.sampleTime(data.eventTime); event.eventTypeId(data.eventType); QJsonObject obj; obj[QStringLiteral("streamName")] = data.streamName; if (data.endTime > 0) { qint64 duration = data.endTime - data.eventTime; obj[QStringLiteral("duration")] = duration; } obj[QStringLiteral("endTime")] = data.endTime; obj[QStringLiteral("MainStructureElementId")] = data.mainStructureElementId; event.properties(QJsonDocument(obj).toJson(QJsonDocument::Compact).toStdString()); return event; } void WebSocketAvs::onTextMessageReceived(const QString &message) { INFO_LOG(m_logger) << "Got message from AVS control process: " << message; const QJsonDocument doc(QJsonDocument::fromJson(message.toUtf8())); const QJsonObject obj = doc.object(); AvsData data = jsonObjectToAvsData(obj); bool lastRangeClosed = obj[QStringLiteral("lastRangeClosed")].toBool(); handleAvsData(data, lastRangeClosed); } void WebSocketAvs::handleAvsData(const AvsData &data, const bool &lastRangeClosed) { QString streamName = data.streamName; DEBUG_LOG(m_logger) << "handleAvsData for stream: " << streamName; auto it = m_avsDataMap.find(streamName); // existing stream .......................................................... if (it != m_avsDataMap.end()) { DEBUG_LOG(m_logger) << "handleAvsData for stream 1: " << streamName; AvsData &avsData = m_avsDataMap[streamName]; if ( (avsData.endTime == -1) && (data.endTime == -1)) { DEBUG_LOG(m_logger) << "kein Publish (avsData.endTime == -1) && (data.endTime == -1)"; return; } if ( (data.endTime == -1) && (data.eventType == ProjectDataManagement::EventType::RadioCnr)) { DEBUG_LOG(m_logger) << "kein Publish (avsData.endTime == -1) && RadioCnr"; return; } if (avsData.eventTime != data.eventTime) // new range -> close old range { DEBUG_LOG(m_logger) << "handleAvsData for stream 2: " << streamName; if (avsData.eventTime == -1) // missing end time -> set artificial end time { WARNING_LOG(m_logger) << "Missing end time for stream: " << streamName << " -> setting artificial end time: " << avsData.eventTime + 1000; avsData.eventTime = avsData.eventTime + 1000; publishData(streamName); } DEBUG_LOG(m_logger) << "handleAvsData for stream remove: " << streamName; m_avsDataMap.remove(streamName); } else // update range { DEBUG_LOG(m_logger) << "handleAvsData update range publish" << streamName; avsData.endTime = data.endTime; publishData(streamName); // if (lastRangeClosed) // m_avsDataMap.remove(streamName); return; } } // new range ................................................................ DEBUG_LOG(m_logger) << "m_avsDataMap.insert(streamName"; m_avsDataMap.insert(streamName, data); AvsData &avsData = m_avsDataMap[streamName]; if (!lastRangeClosed) { DEBUG_LOG(m_logger) << "!lastRangeClosed"; avsData.endTime = -1; } else if (avsData.endTime == -1) // range is closed but end time is missing { DEBUG_LOG(m_logger) << "avsData.endTime == -1"; avsData.endTime = avsData.eventTime + 1000; // -> set artificial end time } DEBUG_LOG(m_logger) << "publishData"; avsData.id = publishData(streamName); // if (lastRangeClosed) // m_avsDataMap.remove(streamName); // // new stream ............................................................... // if (it == m_avsDataMap.end()) // { // DEBUG_LOG(m_logger) << "received event on new stream: " << streamName; // AvsData avsData = data; // if (!lastRangeClosed) // waiting for end time // avsData.endTime = -1; // else if (avsData.endTime == -1) // missing end time -> set artificial end time // avsData.endTime = avsData.eventTime + 1000; // m_avsDataMap.insert(streamName, avsData); // m_avsDataMap[streamName].id = publishData(streamName); // if (lastRangeClosed) // event is over -> data not needed anymore // { // m_avsDataMap.remove(streamName); // return; // } // } // // .......................................................................... // AvsData &avsData = m_avsDataMap[streamName]; // working on reference // if (avsData.eventTime != data.eventTime) // same stream, different time -> new event received // { // DEBUG_LOG(m_logger) << "new event on stream: " << streamName; // // close old event ........................................................ // if (avsData.endTime == -1) // last event on the same stream was not closed // { // WARNING_LOG(m_logger) << "Missing end time for stream: " << streamName << ". Setting artificial end time."; // avsData.endTime = avsData.eventTime + 1000; // publishData(streamName); // } // // ........................................................................ // avsData = data; // avsData.id = 0; // m_avsDataMap[streamName].id = publishData(streamName); // if (lastRangeClosed) // event is over -> data not needed anymore // { // m_avsDataMap.remove(streamName); // return; // } // } // else // same event // { // avsData.endTime = data.endTime; // DEBUG_LOG(m_logger) << "received new end time: " << data.endTime << " for stream: " << streamName; // if (lastRangeClosed) // { // publishData(streamName); // m_avsDataMap.remove(streamName); // event is over -> data not needed anymore // } // } } int WebSocketAvs::publishData(const QString &streamName) { DEBUG_LOG(m_logger) << "WebSocketAvs::publishData"; if (streamName.isEmpty()) { WARNING_LOG(m_logger) << "Ignoring empty stream name"; return 0; } AvsData data = m_avsDataMap.value(streamName); Database::EemEvent dbData = avsDataToEemEvent(data); ExerciseEventManagement::Event ddsEvent = avsDataToEvent(data); qint64 endTime = data.endTime; bool isAlreadyInDb = false; auto cb = [this, dbData, &isAlreadyInDb, streamName, endTime] (const Database::EemEvent &ev) { // endTime and streamName aus DB Event (ev) Properties lesen! const QJsonDocument evDoc = QJsonDocument::fromJson(ev.properties.toUtf8()); const QJsonObject evObj = evDoc.object(); const qint64 evEndTime = evObj[(QStringLiteral("end_time"))].toInt(); const QString streamNameDb = evObj[QStringLiteral("streamName")].toString(); if ((ev.sampleTime == dbData.sampleTime && ev.eventTypeId == dbData.eventTypeId) || (evEndTime > 0 && endTime == -1 )) { if (streamName == streamNameDb) isAlreadyInDb = true; } }; if (!m_dbHandler->db()->selectBySampleTime(cb, dbData.sampleTime)) { if (isAlreadyInDb) { WARNING_LOG(m_logger) << "Ignoring existing event at " << dbData.sampleTime; return 0; } } // write to database if (dbData.id > 0) { if (!m_dbHandler->db()->update(dbData)) ERROR_LOG(m_logger) << "Error while updating dbData with id : " << data.id; } else { if (!m_dbHandler->db()->insert(dbData)) ERROR_LOG(m_logger) << "Error while inserting avsData in db: streamName: " << data.streamName; ddsEvent.id(dbData.id); } // publish data INFO_LOG(m_logger) << "Publish data for stream: " << streamName << " and database id=" << dbData.id; m_eventServer->sendEvent(Database::toJson(dbData)); m_ddsHandler->dts()->publish(ddsEvent); m_mainHandler->addEventIdToRemove(dbData.id); // take data from dds-bus after 10s, we need just to make sure the recording service got it return dbData.id; } void WebSocketAvs::openWebSocket() { DEBUG_LOG(m_logger) << "openWebSocket"; closeWebSocket(); m_wsSocket = new QWebSocket; connect(m_wsSocket, QOverload::of(&QWebSocket::error), this, &WebSocketAvs::onWebSocketError); connect(m_wsSocket, &QWebSocket::connected, this, &WebSocketAvs::onWebSocketConnected); connect(m_wsSocket, &QWebSocket::disconnected, this, &WebSocketAvs::onWebSocketDisconnected); connect(m_wsSocket, &QWebSocket::textMessageReceived, this, &WebSocketAvs::onTextMessageReceived); m_wsSocket->open(QUrl::fromEncoded(m_wsUrl.toLocal8Bit())); } void WebSocketAvs::closeWebSocket() { DEBUG_LOG(m_logger) << "closeWebSocket"; if (m_wsSocket) { m_wsSocket->disconnect(this); m_wsSocket->close(); m_wsSocket->deleteLater(); m_wsSocket = nullptr; } m_wsReconnectTimer->stop(); } void WebSocketAvs::onWebSocketError(QAbstractSocket::SocketError error) { if (!m_wsSocket) return; WARNING_LOG(m_logger) << "Error connecting to " << m_wsSocket->peerAddress().toString() << ": " << error; closeWebSocket(); m_wsReconnectTimer->start(1000); } void WebSocketAvs::onWebSocketConnected() { DEBUG_LOG(m_logger) << "connected to via websockets"; m_wsReconnectTimer->stop(); m_bWebSocketDisconnectedWarnedOnce = false; QTimer::singleShot(300, this, &WebSocketAvs::onSendExerciseId); } void WebSocketAvs::onWebSocketDisconnected() { DEBUG_LOG(m_logger) << "WebSocketAvs::onWebSocketDisconnected()"; if (!m_bWebSocketDisconnectedWarnedOnce) { m_bWebSocketDisconnectedWarnedOnce = true; WARNING_LOG(m_logger) << "Websocket to " << m_wsUrl << " closed - trying to reconnect in 1 sec"; } closeWebSocket(); m_wsReconnectTimer->start(1000); } void WebSocketAvs::onSendExerciseId() { ERROR_LOG(m_logger) << "Send exerciseId: "; QJsonObject obj; obj[QStringLiteral("exerciseId")] = QJsonValue(m_exerciseId); QString jsonStr = QString::fromUtf8(QJsonDocument(obj).toJson()); m_wsSocket->sendTextMessage(jsonStr); } void WebSocketAvs::onCleanUpDbInternal() { m_cleanupTimer->stop(); const auto allDataMap = getAllAvsData(); const auto dbData = getAllInternalDbData(); //qCritical() << " START CLEANUP with " << allDataMap.keys().size() << " keys"; m_avmDtc->waitForHistoricalDataStreamName(1500); m_ismDtc->waitForHistoricalDataMainStructureElement(1500); // qCritical() << "CHECK STREAMNAME " << m_avmDtc->streamNames().size(); for (const auto &stream : allDataMap.keys()) { QVector externalData = allDataMap.value(stream); //This is the fixed map ... there shouldn't be multiple entries! QMap ntpRangeSortExternal; for (const auto &eavItem : externalData) { ntpRangeSortExternal.insert(eavItem.eventTime, eavItem); } QMultiMap ntpRangeSortLocal; for (const auto &avItem : dbData.value(stream)) { ntpRangeSortLocal.insert(avItem.eventTime, avItem); } DEBUG_LOG(m_logger) <<" Start with key: '" << stream << "' external = " << externalData.size() << " internal: " << ntpRangeSortLocal.size(); QVector itemsToRemoveFromInternal; QMap collection; QVector toAdd; QVector toIgnore; QVector toUpdate; for (const auto &eItem : ntpRangeSortExternal) { QList list = ntpRangeSortLocal.values(eItem.eventTime); if (list.isEmpty()) { //there is no external item so add if (!collection.contains(eItem.eventTime)) { AvsData add = eItem; bool checkMsi = eItem.eventType == ProjectDataManagement::EventType::CameraVideo; int mountingPointId = 0; if (checkMsi && m_avmDtc->mountingPointForStreamName(eItem.streamName, mountingPointId)) { //FATAL_LOG(m_logger) << "FOUND MPI "; int mainStructureElementId = 0; if (m_ismDtc->getBuildingId(mountingPointId, mainStructureElementId)) add.mainStructureElementId = mainStructureElementId; } DEBUG_LOG(m_logger) << "need to add: " << add.eventType << " st: " << add.streamName << " mpi:" << add.mainStructureElementId; toAdd.append(add); collection.insert(add.eventTime, add); } continue; } for ( auto item : list) { bool checkMsi = eItem.eventType == ProjectDataManagement::EventType::CameraVideo; if (item.endTime == eItem.endTime ) { //FINF MSE ID! int mountingPointId = 0; int mainStructureElementId = 0; if (checkMsi && m_avmDtc->mountingPointForStreamName(item.streamName, mountingPointId)) { //qCritical() << "FOUND mpi" << mountingPointId << " for Streamname: " << item.streamName; m_ismDtc->getBuildingId(mountingPointId, mainStructureElementId); //qCritical() << "FOUND msi" << mainStructureElementId; } if(item.mainStructureElementId == 0 && mainStructureElementId > 0) { item.mainStructureElementId = mainStructureElementId; toUpdate.append(item); } else toIgnore.append(item); collection.insert(item.eventTime, item); continue; }else // if (eItem.endTime != item.endTime) is always true { item.endTime = eItem.endTime; //item.eventType = lItem.eventType; int mountingPointId = 0; if (checkMsi && m_avmDtc->mountingPointForStreamName(item.streamName, mountingPointId)) { int mainStructureElementId = 0; if (m_ismDtc->getBuildingId(mountingPointId, mainStructureElementId)) { item.mainStructureElementId = mainStructureElementId; } } toUpdate.append(item); collection.insert(item.eventTime, item); } } } for (const AvsData &oldEntry : qAsConst(ntpRangeSortLocal)) { const auto &item = collection.value(oldEntry.eventTime); if (item.id <= 0 ) itemsToRemoveFromInternal.append(oldEntry.id); } //now remove old entries //REMOVE DATA for( const auto &del: itemsToRemoveFromInternal) { // FATAL_LOG(m_logger) << "need to remove " << del; m_dbHandler->db()->removeEemEvent(del); } for( const auto &update: toUpdate) { //UPDATE DATA Database::EemEvent dbDataUpdate = avsDataToEemEvent(update); m_dbHandler->db()->update(dbDataUpdate); } for( const auto &add: toAdd) { //ADD Database::EemEvent dbDataAdd = avsDataToEemEvent(add); dbDataAdd.id = 0; m_dbHandler->db()->insert(dbDataAdd); } DEBUG_LOG(m_logger) << "DONE Sync wite remote DB "<< stream << " to Ignore" << toIgnore.size()<< " to Add" << toAdd.size() << " to Update:" << toUpdate.size()<< " to remove:" << itemsToRemoveFromInternal.size(); } m_cleanupTimer->start(); m_lastUpdate = QDateTime::currentMSecsSinceEpoch()-2*60*1000; DEBUG_LOG(m_logger) << "DONE Sync wite remote DB next check starts at: " << QDateTime::fromMSecsSinceEpoch(m_lastUpdate).toString(); } QMap > WebSocketAvs::getAllAvsData() { //select all Data from AVS avm_timeline events // order by stream QMultiMap ret; QMap > retu; //QString tableNameCameraEvents = (QLatin1String("avm") + QLatin1String(".avs_camera_events")); // todo get shema name QString stmt(QLatin1String("SELECT id, time_start_msec, time_end_msec, stream_name, event_type, last_update FROM ") + QLatin1String("avm.avs_camera_events") + QLatin1String(" WHERE context_id = :context")); if (m_lastUpdate > 0) stmt += QLatin1String(" and time_start_msec > ") + QString::number(m_lastUpdate); DEBUG_LOG(m_logger) << "START Sync read remote DB at " << QDateTime::fromMSecsSinceEpoch(m_lastUpdate).toString(); NSqlQuery q(m_dbHandler->db()->database()); if (!q.prepare(stmt)) { WARNING_LOG(m_logger) << "Could not prepare statement" << stmt << q.lastError().text() << q.lastError().databaseText(); return retu; } // if q.setForwardOnly(true); QString val = QStringLiteral("ex_") + QString::number(m_exerciseId); q.bindValue(QLatin1String(":context"),val); // qWarning() << "frist try statement fallback" << stmt ; if (!q.exec()) { DEBUG_LOG(m_logger) << "Could not execute" << stmt << q.lastError().text() << q.lastError().databaseText(); return retu; } // if //qWarning() << "frist try FOUND" << q.size() ; AvsData item; int j =0; while(q.next()) { int i = 0; item.id = q.value(i++).toInt(); item.eventTime = q.value(i++).toDouble(); item.endTime = q.value(i++).toDouble(); item.streamName = q.value(i++).toString().trimmed(); item.eventType = avsEventTypeToPdmEventType(q.value(i++).toInt()); ret.insert(item.streamName, item); j++; } // while for (const auto &stream: ret.uniqueKeys()) { retu.insert(stream, ret.values(stream).toVector()); } // qCritical() << "DONE reading external db " << j; return retu; } QMap> WebSocketAvs::getAllInternalDbData() { //get all local db data QMap > retu; QMultiMap ret; QVector dataVec; //qCritical() << "START reading internal db "; m_dbHandler->db()->selectByEventTypeId(dataVec, ProjectDataManagement::EventType::CameraVideo); m_dbHandler->db()->selectByEventTypeId(dataVec, ProjectDataManagement::EventType::RadioCnr); m_dbHandler->db()->selectByEventTypeId(dataVec, ProjectDataManagement::EventType::RadioTvc); // m_dbHandler->db()->select(dataVec); //qCritical() << "END select reading internal db "; DEBUG_LOG(m_logger) << "START Sync read local DB at " << QDateTime::fromMSecsSinceEpoch(m_lastUpdate).toString(); int j = 0; for (const auto &dataI : qAsConst(dataVec)) { if (m_lastUpdate > 0 && dataI.sampleTime < m_lastUpdate) continue; QJsonDocument job = QJsonDocument::fromJson(dataI.properties.toUtf8()); if(!job.isObject()) { WARNING_LOG(m_logger) << "ERROR Parsing Json"; } QJsonObject obj = job.object(); AvsData data; data.id = dataI.id; data.eventTime = dataI.sampleTime; data.endTime = obj.value(QStringLiteral("endTime")).toDouble(); data.eventType = dataI.eventTypeId; data.mainStructureElementId =obj.value(QStringLiteral("MainStructureElementId")).toDouble(); data.streamName =obj.value(QStringLiteral("streamName")).toString(); ret.insert(data.streamName, data); j++; //ok add to map } for (const auto &stream: ret.uniqueKeys()) { retu.insert(stream, ret.values(stream).toVector()); } // qCritical() << "DONE reading internal db" << j; return retu; } } // namespace ExerciseEventManagement