/** * @RDE_HEADER@ * Subscribe to file management service * $Id$ * $HeadURL$ * $Author$ * * Created by Norbert Thoden, NLE, 2019-04-15 */ #include "asb_flm_subscriber.h" #include #include #include #include #include "flm_symbol_proxy.h" #include #include // Qt #include #include #include #include #include #include #include #include #include namespace AdvancedSymbology { uint qHash(SymbologyType type) { return static_cast(type); } struct FileContentStruct { enum SymbolType { UnsupportedSymbol, EtsSymbol, XmlSymbol }; enum ReplyState { Undefined, Requested, RequestFailure, ReplyReceived, ReplySuccess, ReplyFailure }; QString name; SymbolType type{UnsupportedSymbol}; QString operatorName; int64_t lastModified{0}; QStringList usrTags; QColor colorTobeReplaced; struct Monitoring { qint64 requestTime{-1}; // sec int retryCnt{0}; ReplyState replyState{Undefined}; int replyContentSize{-1}; }; mutable Monitoring monitoring; }; qint64 getRequestTime() { return time(nullptr); } QString toString(FileContentStruct::ReplyState rs) { switch (rs) { case FileContentStruct::Undefined: return QStringLiteral("Undefined"); case FileContentStruct::Requested: return QStringLiteral("Requested"); case FileContentStruct::RequestFailure: return QStringLiteral("RequestFailure"); case FileContentStruct::ReplyReceived: return QStringLiteral("ReplyReceived"); case FileContentStruct::ReplySuccess: return QStringLiteral("ReplySuccess"); case FileContentStruct::ReplyFailure: return QStringLiteral("ReplyFailure"); } return QStringLiteral("UnknownReplyState"); } //------------------------------------------------------------------------------ struct FlmSubscriber::Private { QHash m_fileContentMap; // fileContentId -> name QTimer *m_reFetchTimer{nullptr}; FlmSymbolProxy m_symbolProxy; QSet m_symbolDictionaryToUpdate; std::shared_ptr m_flmDtc; std::shared_ptr m_flmFtc; std::shared_ptr m_opmDtc; static long m_counter; }; long FlmSubscriber::Private::m_counter = 0; FlmSubscriber::FlmSubscriber(const std::shared_ptr &flmDtc, const std::shared_ptr &flmFtc, const std::shared_ptr &opmDtc) : m_logger(NGL::NGLLoggerBootstrap::getNGLRootLogger().getNGLLogger("AdvSymbology.FlmSubscriber")) , d(new Private()) { INFO_LOG(m_logger) << LOGFUNCTION << " m_counter =" << ++d->m_counter; d->m_flmDtc = flmDtc; d->m_flmFtc = flmFtc; d->m_opmDtc = opmDtc; d->m_reFetchTimer = new QTimer(this); d->m_reFetchTimer->setInterval(10000); connect(d->m_reFetchTimer, &QTimer::timeout, this, &FlmSubscriber::checkForReFetching); // To enable QueuedConnection: qRegisterMetaType("FileManagement::FileContent"); qRegisterMetaType("DataModification"); } FlmSubscriber::~FlmSubscriber() { INFO_LOG(m_logger) << LOGFUNCTION << " m_counter =" << d->m_counter--; if (d->m_reFetchTimer) d->m_reFetchTimer->stop(); delete d; } bool FlmSubscriber::createFlmSymbolProxy(int type_placeholder) { Q_UNUSED(type_placeholder); if (!d->m_flmDtc || !d->m_flmFtc) return false; d->m_flmDtc->enableFileContent(); d->m_flmFtc->enableGetFileContentData(); // // 1 connect changed signals: // // avoid multiple connect connect(d->m_flmDtc.get(), &FileManagement::Interface::Dtc::fileContentChanged, this, &FlmSubscriber::onFileContentChanged, Qt::ConnectionType(Qt::QueuedConnection | Qt::UniqueConnection)); // // 2. retrieve FileContent // const auto cb = [this](const FileManagement::FileContent & data) -> void { updateFileContentMap(data); }; if (!d->m_flmDtc->fileContents(cb)) ERROR_LOG(m_logger) << LOGFUNCTION << " Error performing fileContents"; INFO_LOG(m_logger) << LOGFUNCTION << " m_fileContentMap.size()=" << d->m_fileContentMap.size() ; // // now we have all FileContentIds: -> retrieve contentDATA (svg data) // for (auto it = d->m_fileContentMap.cbegin(); it != d->m_fileContentMap.cend(); ++it) { const int fileContentId = it.key(); FileManagement::GetFileContentDataRequest request(fileContentId); const FileContentStruct &fcsRef = it.value(); // monitoring is mutable #ifdef LTS_DEBUG //#define SKIP_SOME_TO_TEST_REFETCHING #ifdef SKIP_SOME_TO_TEST_REFETCHING if (fileContentId>=98) request.fileContentId(-1); #endif // SKIP_SOME_TO_TEST_REFETCHING #endif // LTS_DEBUG if (d->m_flmFtc->getFileContentDataRequest(request, this)) { fcsRef.monitoring.requestTime = getRequestTime(); fcsRef.monitoring.replyState = FileContentStruct::Requested; } else { fcsRef.monitoring.replyState = FileContentStruct::RequestFailure; ERROR_LOG(m_logger) << "Error requesting FileContentData, fileContentId=" << fileContentId; } } // while // // start reFetchTimer using rand to avoid heavy load at server side: // const int randomOffset = rand() % 10 + 5; // 5 .. 15 QTimer::singleShot(randomOffset, this , [this]() { DEBUG_LOG(m_logger) << "start ReFetchingTimer"; d->m_reFetchTimer->start(); // -> checkForReFetching }); return true; } void FlmSubscriber::onGetFileContentDataReply(const FileManagement::GetFileContentDataReply &reply) { const int32_t fileContentId = reply.id(); TRACE_LOG(m_logger) << LOGFUNCTION << " START, " << fileContentId; if (!d->m_fileContentMap.contains(fileContentId)) { ERROR_LOG(m_logger) << LOGFUNCTION << " fileContentId unknown, fileContentId=" << fileContentId; return; } const FileContentStruct &fcData = d->m_fileContentMap[fileContentId]; fcData.monitoring.replyState = FileContentStruct::ReplyReceived; const QString &name = fcData.name; const QString &operatorName = fcData.operatorName; const qint64 lastModified = fcData.lastModified; DEBUG_LOG(m_logger) << LOGFUNCTION << " fileContentId=" << fileContentId << " name=" << name << " operatorName=" << operatorName << " lastModified=" << lastModified << " #bytes =" << reply.content().size(); fcData.monitoring.replyContentSize = reply.content().size(); // update ReplayState switch (fcData.type) { case FileContentStruct::SymbolType::EtsSymbol: { QImage image; if (!svgToImage(QByteArray::fromRawData(reinterpret_cast(&reply.content().at(0)), reply.content().size()), image)) { ERROR_LOG(m_logger) << LOGFUNCTION << " svgToImage failed, fileContentId=" << fileContentId; fcData.monitoring.replyState = FileContentStruct::ReplyFailure; // update ReplayState return; } // now we have metaData (fileContentId, name) and data (content/svg) // transfer to symbolProxy: FlmSymbolProxy::SymbolData symbolData(name, image, fcData.colorTobeReplaced); symbolData.setUsrTags(fcData.usrTags); // store: d->m_symbolProxy.setSymbol(fileContentId, symbolData); fcData.monitoring.replyState = FileContentStruct::ReplySuccess; // update ReplayState break; } case FileContentStruct::SymbolType::XmlSymbol: { FlmSymbolProxy::SymbolData symbolData(name, operatorName, lastModified, LtsUtils::toQByteArray(reply.content())); if (symbolData.isXmlSymbol()) { symbolData.setUsrTags(fcData.usrTags); // store it d->m_symbolProxy.setSymbol(fileContentId, symbolData); fcData.monitoring.replyState = FileContentStruct::ReplySuccess; // update ReplayState } else { ERROR_LOG(m_logger) << LOGFUNCTION << " xmlSymbolContent is empty, fileContentId=" << fileContentId; fcData.monitoring.replyState = FileContentStruct::ReplyFailure; // update ReplayState } break; } case FileContentStruct::SymbolType::UnsupportedSymbol: { ERROR_LOG(m_logger) << LOGFUNCTION << " unsupported symbol found, fileContentId=" << fileContentId; fcData.monitoring.replyState = FileContentStruct::ReplyFailure; // update ReplayState break; } } // switch TRACE_LOG(m_logger) << LOGFUNCTION << " END"; } bool FlmSubscriber::svgToImage(const QByteArray &svgByteArray, QImage &imageOut) { TRACE_LOG(m_logger) << LOGFUNCTION << " START"; // load QByteArray QSvgRenderer svgRenderer; if (!svgRenderer.load(svgByteArray)) { ERROR_LOG(m_logger) << LOGFUNCTION << " svgRenderer.load failed"; return false; } // prepare Image: int size = 200; // pixel imageOut = QImage(size, size, QImage::Format_ARGB32); imageOut.fill(Qt::transparent); // render QPainter painter(&imageOut); svgRenderer.render(&painter); painter.end(); TRACE_LOG(m_logger) << LOGFUNCTION << " END"; return true; } QString FlmSubscriber::operatorName(int operatorId) { if (d->m_opmDtc) return QString::fromStdString(d->m_opmDtc->user(operatorId).name()); return QStringLiteral("unknown user<%1>").arg(operatorId); } void FlmSubscriber::checkForReFetching() { // use own logging channel: LOGGER_T logger = NGL::NGLLoggerBootstrap::getNGLRootLogger().getNGLLogger("AdvSymbology.FlmSubscriber.ReFetch"); DEBUG_LOG(logger) << "********** checkForReFetching START, size=" << d->m_fileContentMap.size(); int i{0}; int unfinished{0}; int reFetchCounter{0}; static int s_reFetchCounter = 0; const qint64 curTime = time(nullptr); for (auto it = d->m_fileContentMap.cbegin(); it != d->m_fileContentMap.cend(); ++it) { const FileContentStruct &fcs = it.value(); qint64 ageSec = curTime - fcs.monitoring.requestTime; DEBUG_LOG(logger) << "***** dump i=" << ++i << ", fileContentId=" << it.key() << ", retryCnt=" << fcs.monitoring.retryCnt << ", ageSec = " << ageSec << ", replyState=" << toString(fcs.monitoring.replyState) << ", replyContentSize=" << fcs.monitoring.replyContentSize << ", name='" << fcs.name << "'" ; if (fcs.monitoring.replyState != FileContentStruct::ReplySuccess) { ++unfinished; if ((ageSec > 30) && (fcs.monitoring.retryCnt < 3)) { // retry / reFetch const int32_t fileContentId = it.key(); INFO_LOG(logger) << "***** reFetch *** :" << " ageSec = " << ageSec << ", fileContentId=" << fileContentId << ", replyState=" << toString(fcs.monitoring.replyState) << ", replyContentSize=" << fcs.monitoring.replyContentSize << ", name='" << fcs.name << "'" ; FileManagement::GetFileContentDataRequest request(fileContentId); if (!d->m_flmFtc->getFileContentDataRequest(request, this)) // -> onGetFileContentDataReply -> ... -> setSymbol ERROR_LOG(logger) << LOGFUNCTION << "Error reading FileContentData, fileContentId=" << fileContentId; fcs.monitoring.replyState = FileContentStruct::Requested; fcs.monitoring.requestTime = getRequestTime(); ++fcs.monitoring.retryCnt; // Server-Restart: Topic is removed and therefore the fcs is also removed from container -> retryCnt is fine ++reFetchCounter; ++s_reFetchCounter; } } } INFO_LOG(logger) << "***** checkForReFetching END" << ", size=" << d->m_fileContentMap.size() << ", unfinished=" << unfinished << ", reFetchCounter=" << reFetchCounter << ", s_reFetchCounter=" << s_reFetchCounter; } void FlmSubscriber::onFileContentChanged(const FileManagement::FileContent &data, DataModification modType) { TRACE_LOG(m_logger) << LOGFUNCTION << " data.id=" << data.id() << " data.fileStatus=" << data.fileStatus().underlying() << " modType=" << modType << " data.name='" << data.name().c_str() << "'"; if (!d->m_flmFtc) return; if (data.isDeleted()) return; // check fileType if (data.fileType().underlying() != FileManagement::FileType::OperatorDefinedSymbol) return; // check fileStatus in relation with modType: // (if modified to FileStatus::Deleted process as DataModification::eRemoved) if (data.fileStatus().underlying() == FileManagement::FileStatus::Deleted) { switch (modType) { case DataModification::eAdded: return; case DataModification::eModified: modType = DataModification::eRemoved; break; case DataModification::eRemoved: // empty break; } // switch } // // no check against ExerciseId, because we assume that all symbols are 'global' // const int32_t fileContentId = data.id(); switch (modType) { case DataModification::eAdded: case DataModification::eModified: { // since it´s caused by an update/changed signal, // we have to update our own map, (to let onGetFileContentDataReply work) updateFileContentMap(data); FileManagement::GetFileContentDataRequest request(fileContentId); if (!d->m_flmFtc->getFileContentDataRequest(request, this)) // -> onGetFileContentDataReply -> ... -> setSymbol ERROR_LOG(m_logger) << LOGFUNCTION << "Error reading FileContentData, fileContentId=" << fileContentId; break; } case DataModification::eRemoved: { DEBUG_LOG(m_logger) << LOGFUNCTION << " modType=" << modType << " fileContentId=" << fileContentId << " fileContentName=" << data.name(); // update own container: d->m_fileContentMap.remove(fileContentId); // remove: if (!d->m_symbolProxy.removeSymbol(fileContentId)) ERROR_LOG(m_logger) << LOGFUNCTION << " removeSymbol failed" << " fileContentId=" << fileContentId << " fileContentName='" << data.name() << "'" << " contains=" << d->m_fileContentMap.contains(fileContentId); break; } } // switch } // void FlmSubscriber::onFileContentChanged(const FileManagement::FileContent &data, DataModification modType) void FlmSubscriber::updateFileContentMap(const FileManagement::FileContent &data) { TRACE_LOG(m_logger) << LOGFUNCTION << " START" << " d->m_fileContentMap.size()=" << d->m_fileContentMap.size() << " fileContentId=" << data.id() << " data.isDeleted()=" <errorString: '" << parseError.errorString() << "'" << ", offset: " << parseError.offset; } // if (parseError.error == QJsonParseError::NoError) } // if (data.attributes().is_set()) d->m_fileContentMap[fileContentId] = fcData; TRACE_LOG(m_logger) << LOGFUNCTION << " END" << " d->m_fileContentMap.size()=" << d->m_fileContentMap.size() << " fileContentId=" << data.id() << " data.isDeleted()=" <m_symbolDictionaryToUpdate)) { AdvancedSymbology::SymbolDictionary &dict = AdvancedSymbology::SymbolDictionary::dictionary(symbologyType); this->setEtsSymbolToDict(dict, fileContentId, name, image, colorToBeReplaced, usrTags); } } void FlmSubscriber::onLocalSymbolProxyEtsSymbolRemoved(qint32 fileContentId, const QString &name) { for (const SymbologyType &symbologyType : qAsConst(d->m_symbolDictionaryToUpdate)) { AdvancedSymbology::SymbolDictionary &dict = AdvancedSymbology::SymbolDictionary::dictionary(symbologyType); INFO_LOG(m_logger) << LOGFUNCTION << " dict.removeEtsSymbol:" << " symbologyType='" << (int)symbologyType << "'" << ", name='" << name << "'" << ", fileContentId=" << fileContentId; dict.removeEtsSymbol(fileContentId); dict.removeEtsSymbol(name); } } void FlmSubscriber::onLocalSymbolProxyXmlSymbolAdded(qint32 fileContentId, const QString &operatorName, qint64 lastModified, const QString &name, const QString &xmlContent, const QStringList &usrTags) { TRACE_LOG(m_logger) << LOGFUNCTION << " START" << " fileContentId=" << fileContentId << " name='" << name << "'"; if (!xmlContent.isEmpty()) { for (const SymbologyType &symbologyType : qAsConst(d->m_symbolDictionaryToUpdate)) { AdvancedSymbology::SymbolDictionary &dict = AdvancedSymbology::SymbolDictionary::dictionary(symbologyType); if (!this->setXmlSymbolToDict(dict, fileContentId, operatorName, lastModified, name, xmlContent, usrTags)) ERROR_LOG(m_logger) << LOGFUNCTION << " dict.setXmlSymbolToDict failed" << ", symbologyType='" << (int)symbologyType << "'" << ", name='" << name << "'"; } } TRACE_LOG(m_logger) << LOGFUNCTION << " END" << " fileContentId=" << fileContentId << " name='" << name << "'"; } void FlmSubscriber::onLocalSymbolProxyXmlSymbolRemoved(qint32 fileContentId) { TRACE_LOG(m_logger) << LOGFUNCTION << " START" << " fileContentId=" << fileContentId; for (const SymbologyType &symbologyType : qAsConst(d->m_symbolDictionaryToUpdate)) { AdvancedSymbology::SymbolDictionary &dict = AdvancedSymbology::SymbolDictionary::dictionary(symbologyType); DEBUG_LOG(m_logger) << LOGFUNCTION << " dict.removeXmlSymbol:" << " symbologyType='" << symbologyType << "'" << ", fileContentId=" << fileContentId; const QString &skey = QString::number(fileContentId); const QString &sidc = SidcCodec::encodeCivilXml(skey); dict.removeXmlSymbol(sidc); dict.removeXmlSymbol(fileContentId); } TRACE_LOG(m_logger) << LOGFUNCTION << " END" << " fileContentId=" << fileContentId; } // // Proxy // bool FlmSubscriber::connectFlmSymbolProxy(int type_placeholder, SymbologyType symbologyType) { Q_UNUSED(type_placeholder); TRACE_LOG(m_logger) << LOGFUNCTION << " m_fileContentMap.size()=" << d->m_fileContentMap.size() ; d->m_symbolDictionaryToUpdate.insert(symbologyType); // ETS // connect to some signals like added, modified, removed: connect(&d->m_symbolProxy, &FlmSymbolProxy::etsSymbolAdded, this, &FlmSubscriber::onLocalSymbolProxyEtsSymbolAdded, Qt::QueuedConnection); connect(&d->m_symbolProxy, &FlmSymbolProxy::etsSymbolRemoved, this, &FlmSubscriber::onLocalSymbolProxyEtsSymbolRemoved, Qt::QueuedConnection); // XML // connect to some signals like added, modified, removed: connect(&d->m_symbolProxy, &FlmSymbolProxy::xmlSymbolAdded, this, &FlmSubscriber::onLocalSymbolProxyXmlSymbolAdded, Qt::QueuedConnection); connect(&d->m_symbolProxy, &FlmSymbolProxy::xmlSymbolRemoved, this, &FlmSubscriber::onLocalSymbolProxyXmlSymbolRemoved, Qt::QueuedConnection); // continue? if (d->m_symbolProxy.isEmpty()) return true; AdvancedSymbology::SymbolDictionary &dict = AdvancedSymbology::SymbolDictionary::dictionary(symbologyType); // Don´t use QSignalBlocker here!!! dict.blockSignals(true); // to make sure that set{Ets,Xml}SymbolToDict will NOT emit huge number of 'symbolAdded' for (auto mapIt = d->m_symbolProxy.getMap().begin(); mapIt != d->m_symbolProxy.getMap().end(); ++mapIt) { const int fileContentId = mapIt.key(); const FlmSymbolProxy::SymbolData &symData = mapIt.value(); if (symData.isXmlSymbol()) setXmlSymbolToDict(dict, fileContentId, symData.operatorName(), symData.lastModified(), symData.name(), symData.xmlContent(), symData.usrTags()); else setEtsSymbolToDict(dict, fileContentId, symData.name(), symData.image(), symData.colorToBeReplaced(), symData.usrTags()); } dict.blockSignals(false); return true; } bool FlmSubscriber::setEtsSymbolToDict(SymbolDictionary &dict, qint32 fileContentId, const QString &name, const QImage &image, const QColor &colorToBeReplaced, const QStringList &usrTags) { // -> ETS_%1 SymbolProperties symProps; symProps.setKey(name); symProps.setSidc(SidcCodec::encodeCivilEts(name)); symProps.setName(name); // symProps.setCategory(); // maybe? symProps.setSymbolClass(QLatin1String("3")); // 18.10.2016, N.Thoden: 3 == PointSymbol (mil2525d.stylx - QuartzRelease) symProps.setGeometryType(GeometryType::Point); // ETS-Symbols are always Points symProps.setTags(QStringList() << name << usrTags); symProps.setColorToBeReplaced(colorToBeReplaced); if (!dict.setEtsSymbol(fileContentId, symProps.sidc(), symProps, image)) { ERROR_LOG(m_logger) << LOGFUNCTION << " dict.insertExtraSymbol failed" << ", fileContentId=" << fileContentId << ", name='" << name << "'"; return false; } else INFO_LOG(m_logger) << LOGFUNCTION << " dict.insertExtraSymbol done" << ", fileContentId=" << fileContentId << ", name='" << name << "'"; return true; } bool FlmSubscriber::setXmlSymbolToDict(SymbolDictionary &dict, qint32 fileContentId, const QString &operatorName, qint64 lastModified, const QString &name, const QString &xmlContent, const QStringList &usrTags) { SymbolProperties symProps; /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * ToDo add more search infos to symProps * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ symProps.setTags(QStringList() << name << usrTags); bool success = dict.setXmlSymbol(fileContentId, name, operatorName, lastModified, xmlContent, symProps); if (!success) ERROR_LOG(m_logger) << LOGFUNCTION << " setXmlSymbol failed:" << ", file-id=" << fileContentId << ", filenameid='" << name << "'"; return success; } } // namespace AdvancedSymbology