/** * @RDE_HEADER@ * CodificationDataManagement - Import codification table csv data * $Id$ * $HeadURL$ * $Author$ * Created by hjl, SLSE, 2017-03-02 */ #include "cod_csv_import.h" #include "cod_numeral_code_model.h" #include "cod_authentication_code_model.h" #include "cod_logistical_terms_model.h" #include "cod_phrase_table_model.h" #include #include #include #include using namespace CodificationManagement; CSVImport::CSVImport(NumeralCodeModel *numCodeModel, AuthenticationCodeModel *authenticationCodeModel, LogisticalTermsModel *logisticalTermsModel, PhraseTableModel *phraseTableModel, QString *note, QWidget *parent) : QWidget(parent) , m_numCodeModel(numCodeModel) , m_authenticationCodeModel(authenticationCodeModel) , m_logisticalTermsModel(logisticalTermsModel) , m_phraseTableModel(phraseTableModel) , m_note(note) , m_logger(getLogger("CodificationCSVImport")) { } // end of ctor() //----------------------------------------------------------------------------- CSVImport::CSVImportResult CSVImport::loadCSVDataByteArray(QByteArray content) { QBuffer buffer; buffer.setBuffer(&content); buffer.open(QIODevice::ReadOnly); QByteArray qbaNumCodeHorizontalHeader; // QByteArray qbaNumCodeVerticalHeader; // QByteArray qbaAuthenticationCodeMatrix; // QByteArray qbaLogTermsTableHorizontalHeader; // QByteArray qbaLogTermsTableVerticalHeader; // QByteArray qbaLogTermsTableMatrix; // QByteArray qbaPhraseTableMatrix; // QByteArray qbaNote; // QByteArray *qba; qba = nullptr; int desiredCvsSeperatorsPerLine = 0; int csvKeywordCount = 0; QByteArray qbaMergeQuotedEntriesPerLine; bool isMergedLine = false; while (!buffer.atEnd()) { QByteArray qbaLine = buffer.readLine(); // '\n' in quoted Eintraegen wird hier als Line Ende interpretiert!! // DEBUG_LOG(m_logger) << "qbaLine (raw): \"" << qbaLine << "\""; // WindowsEnd -> UnixEnd if (qbaLine.endsWith("\r\n")) qbaLine.remove(qbaLine.lastIndexOf('\r'), 1); if (qbaLine.startsWith("<")) { // alle ; Zeichen nach > abschneiden int index = qbaLine.indexOf('>'); qbaLine.truncate(index + 1); qbaLine.append('\n'); if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaNumCodeHorizontalHeader; desiredCvsSeperatorsPerLine = 12; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaNumCodeVerticalHeader; desiredCvsSeperatorsPerLine = 0; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaAuthenticationCodeMatrix; desiredCvsSeperatorsPerLine = 4; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaLogTermsTableHorizontalHeader; desiredCvsSeperatorsPerLine = 4; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaLogTermsTableVerticalHeader; desiredCvsSeperatorsPerLine = 0; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaLogTermsTableMatrix; desiredCvsSeperatorsPerLine = 4; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaPhraseTableMatrix; desiredCvsSeperatorsPerLine = 1; ++csvKeywordCount; continue; } if (qbaLine == "\n") { DEBUG_LOG(m_logger) << "found " << qbaLine; qba = &qbaNote; desiredCvsSeperatorsPerLine = 0; ++csvKeywordCount; continue; } } if (qba) { if (! qbaLine.contains('"') && ! isMergedLine) { // Zeile komplett uebernehmen ... removeRedundantCvsSeperators(qbaLine, desiredCvsSeperatorsPerLine); qba->append(qbaLine); continue; } // moegliche Varianten (Betrachtung speziell fuer LogTermsTable) : // a.) Zeile enthaelt (oeffnendes) " =>> isMergedLine = true setzen -> Zeile in Buf zwischenspeichern -> continue // ... kann hier nur ein " enthalten, da der quoted Anteil mit \n endet ! // b.) naechste zeile enthaelt kein " =>> da isMergedLine == true, sind es quoted Daten -> Daten an Buf anhaengen -> continue // c.) oder Zeile enthaelt (schliessendes) " =>> Daten bis " an Buf anhaengen // ... und ggf weiteres (oeffnendes) " =>> Pruefen ob weiteres (oeffnendes) " vorhanden (das kann sein, wenn die Excelzeile weitere quoted Eintraege enthaelt) // - wenn ja, restliche Zeichen an Buf anhaengen -> continue // - wenn nein, restliche Zeichen an Buf anhaengen -> Zeile komplett... // Gesammelte Zeile in qbaLine uebernehmen // qbaLine = qbaMergeQuotedEntriesPerLine; // qbaMergeQuotedEntriesPerLine.clear(); // isMergedLine = false setzen // removeRedundantCvsSeperators(qbaLine, desiredCvsSeperatorsPerLine); // qba->append(qbaLine); // a.) if (qbaLine.contains('"') && ! isMergedLine) { isMergedLine = true; qbaMergeQuotedEntriesPerLine.append(qbaLine); continue; } // b.) mehrzeiliges Quote... if (! qbaLine.contains('"') && isMergedLine) { qbaMergeQuotedEntriesPerLine.append(qbaLine); continue; } // c.) if (qbaLine.contains('"') && isMergedLine) { // Daten bis incl. " an Buf anhaengen int idx = qbaLine.indexOf('\"'); QByteArray qbaQuotedPart = qbaLine.left(idx + 1); // incl. " qbaMergeQuotedEntriesPerLine.append(qbaQuotedPart); // Rest der Zeile nach schliessenden " QByteArray qbaRestLine = qbaLine.mid(idx + 1); // Pruefen ob im Rest noch weiteres (oeffnendes) " vorhanden if (! qbaRestLine.contains('"')) { // Gesamte CSV-Zeile fertig qbaMergeQuotedEntriesPerLine.append(qbaRestLine); // Gesammelte Zeile in qbaLine uebernehmen qbaLine = qbaMergeQuotedEntriesPerLine; // Merge Modus beenden und fuer naechsten vorbereiten qbaMergeQuotedEntriesPerLine.clear(); isMergedLine = false; // Zeile komplett uebernehmen ... removeRedundantCvsSeperators(qbaLine, desiredCvsSeperatorsPerLine); qba->append(qbaLine); continue; } else { // Rest enthaelt noch ein oeffnendes ". Rest der Zeile an Buf anhaengen qbaMergeQuotedEntriesPerLine.append(qbaRestLine); continue; } } } // if (qba) } // while (!buffer.atEnd()) if (csvKeywordCount != 8) { ERROR_LOG(m_logger) << "Load csv data failed! Invalid csvKeywordCount. Soll/Ist: 8/" << csvKeywordCount; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data failed! Invalid number of cvs keywords."), tr("CSV Load Error")); return CSVImportResult::LoadFailed; } // show load csv data blocks DEBUG_LOG(m_logger) << "Load csv ready... "; DEBUG_LOG(m_logger) << "\n-----------start show load csv data blocks -------------------------\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaNumCodeHorizontalHeader << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaNumCodeVerticalHeader << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaAuthenticationCodeMatrix << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaLogTermsTableHorizontalHeader << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaLogTermsTableVerticalHeader << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaLogTermsTableMatrix << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaPhraseTableMatrix << "\"\n"; DEBUG_LOG(m_logger) << "\n: \"" << qbaNote << "\"\n"; DEBUG_LOG(m_logger) << "\n-----------end of show load csv data blocks -------------------------\n"; // Numeral Code table qbaNumCodeHorizontalHeader = qbaNumCodeHorizontalHeader.trimmed(); // remove last `\n` if (checkNumCodeHorizontalHeader(qbaNumCodeHorizontalHeader)) m_numCodeModel->setColumnSelectionValuesCSV(QString::fromUtf8(qbaNumCodeHorizontalHeader)); else return CSVImportResult::LoadFailed; qbaNumCodeVerticalHeader = qbaNumCodeVerticalHeader.trimmed(); // remove last `\n` if (checkNumCodeVerticalHeader(qbaNumCodeVerticalHeader)) m_numCodeModel->setRowSelectionValuesCSV(QString::fromUtf8(qbaNumCodeVerticalHeader)); else return CSVImportResult::LoadFailed; // Authentication table qbaAuthenticationCodeMatrix = qbaAuthenticationCodeMatrix.trimmed(); // remove last `\n` if (checkAuthenticationCodeMatrix(qbaAuthenticationCodeMatrix)) m_authenticationCodeModel->setMatrixValuesCSV(QString::fromUtf8(qbaAuthenticationCodeMatrix)); else return CSVImportResult::LoadFailed; // Logistical Terms table qbaLogTermsTableHorizontalHeader = qbaLogTermsTableHorizontalHeader.trimmed(); // remove last `\n` if (checkLogTermsTableHorizontalHeader(qbaLogTermsTableHorizontalHeader)) m_logisticalTermsModel->setColumnSelectionValuesCSV(QString::fromUtf8(qbaLogTermsTableHorizontalHeader)); else return CSVImportResult::LoadFailed; qbaLogTermsTableVerticalHeader = qbaLogTermsTableVerticalHeader.trimmed(); // remove last `\n` if (checkLogTermsTableVerticalHeader(qbaLogTermsTableVerticalHeader)) m_logisticalTermsModel->setRowSelectionValuesCSV(QString::fromUtf8(qbaLogTermsTableVerticalHeader)); else return CSVImportResult::LoadFailed; qbaLogTermsTableMatrix = qbaLogTermsTableMatrix.trimmed(); // remove last `\n` if (checkLogTermsTableMatrix(qbaLogTermsTableMatrix)) m_logisticalTermsModel->setMatrixValuesCSV(QString::fromUtf8(qbaLogTermsTableMatrix)); else return CSVImportResult::LoadFailed; // Phrase table qbaPhraseTableMatrix = qbaPhraseTableMatrix.trimmed(); // remove last `\n` if (checkPhraseTableMatrix(qbaPhraseTableMatrix)) m_phraseTableModel->setMatrixValuesCSV(QString::fromUtf8(qbaPhraseTableMatrix)); else return CSVImportResult::LoadFailed; // Note qbaNote = qbaNote.trimmed(); // remove last `\n` removeQuotedCharInNote(qbaNote); m_note->clear(); m_note->append(QString::fromUtf8(qbaNote)); return CSVImportResult::LoadOk; } CSVImport::CSVImportResult CSVImport::loadCSVData() { DEBUG_LOG(m_logger) << "loadCSVData()"; QFileDialog dialog; dialog.setNameFilter(tr("CSV File (*.csv)")); dialog.setAcceptMode(QFileDialog::AcceptOpen); dialog.setDirectory(QLatin1String("../TMP")); if (dialog.exec() == QDialog::Rejected) return CSVImportResult::UserAbort; const QStringList fileNames = dialog.selectedFiles(); if (fileNames.isEmpty()) return CSVImportResult::LoadFailed; QFile file; file.setFileName(fileNames.first()); if (!file.open(QFile::ReadOnly)) { LtsWidgets::LtsMessageDialog::showMessage(this, tr("File is not accessible!"), tr("CSV Import Error")); return CSVImportResult::LoadFailed; } QByteArray byteArray = file.readAll(); file.close(); ////////////////////// loadCSVDataByteArray(byteArray); return CSVImportResult::LoadOk; } // end of loadCSVData() //----------------------------------------------------------------------------- bool CSVImport::checkNumCodeHorizontalHeader(const QByteArray &qbaNumCodeHorizontalHeader) { // Soll: 13 Zeichen in einer Zeile getrennt durch `;` DEBUG_LOG(m_logger) << "Data: \"" << qbaNumCodeHorizontalHeader << "\""; DEBUG_LOG(m_logger) << "Data.size: Soll/Ist: 13 + 12 = 25/" << qbaNumCodeHorizontalHeader.size(); if (qbaNumCodeHorizontalHeader.size() == 25) return true; ERROR_LOG(m_logger) << "Load csv data 'Numeral Code Horizontal Header' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Numeral Code Horizontal Header' failed!"), tr("CSV Load Error")); return false; } // end of checkNumCodeHorizontalHeader() //----------------------------------------------------------------------------- bool CSVImport::checkNumCodeVerticalHeader(const QByteArray &qbaNumCodeVerticalHeader) { // Soll: 13 Zeichen jeweils abgeschlossen mit `\n` (last `\n` removed) DEBUG_LOG(m_logger) << "Data: \"" << qbaNumCodeVerticalHeader << "\""; DEBUG_LOG(m_logger) << "Data.size: Soll/Ist: 13 + 12 = 25/" << qbaNumCodeVerticalHeader.size(); if (qbaNumCodeVerticalHeader.size() == 25) return true; ERROR_LOG(m_logger) << "Load csv data 'Numeral Code Vertical Header' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Numeral Code Vertical Header' failed!"), tr("CSV Load Error")); return false; } // end of checkNumCodeVerticalHeader() //----------------------------------------------------------------------------- bool CSVImport::checkAuthenticationCodeMatrix(QByteArray &qbaAuthenticationCodeMatrix) { // Soll: 13 Zeilen mit je 5x2 Zeichen getrennt durch `;` Zeile mit `\n` abgeschlossen (last `\n` removed) DEBUG_LOG(m_logger) << "Data: \"" << qbaAuthenticationCodeMatrix << "\""; QByteArray qbaOutMatrix; QList qbaLinesList = qbaAuthenticationCodeMatrix.split('\n'); for (QByteArray &line : qbaLinesList) { QList qbaLineList = line.split(';'); for (QByteArray &segment : qbaLineList) { if (segment.size() == 1) // fuehrende Null hinzufuegen segment = segment.rightJustified(2, '0'); } line = qbaLineList.join(';'); } qbaOutMatrix = qbaLinesList.join('\n'); qbaAuthenticationCodeMatrix = qbaOutMatrix; DEBUG_LOG(m_logger) << "Data.size: Soll/Ist: 13 * (5*2 + 4 + 1) -1 = 194 /" << qbaAuthenticationCodeMatrix.size(); if (qbaAuthenticationCodeMatrix.size() == 194) return true; else { ERROR_LOG(m_logger) << "Load csv data 'Authentication Code Matrix' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Authentication Code Matrix' failed!"), tr("CSV Load Error")); return false; } } // end of checkAuthenticationCodeMatrix() //----------------------------------------------------------------------------- bool CSVImport::checkLogTermsTableHorizontalHeader(const QByteArray &qbaLogTermsTableHorizontalHeader) { // Soll: 5 Zeichen in einer Zeile getrennt durch `;` DEBUG_LOG(m_logger) << "Data: \"" << qbaLogTermsTableHorizontalHeader << "\""; DEBUG_LOG(m_logger) << "Data.size: Soll/Ist: 5 + 4 = 9/" << qbaLogTermsTableHorizontalHeader.size(); if (qbaLogTermsTableHorizontalHeader.size() == 9) return true; else { ERROR_LOG(m_logger) << "Load csv data 'Logistical Terms Table Horizontal Header' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Logistical Terms Table Horizontal Header' failed!"), tr("CSV Load Error")); return false; } } // end of checkLogTermsTableHorizontalHeader() //----------------------------------------------------------------------------- bool CSVImport::checkLogTermsTableVerticalHeader(const QByteArray &qbaLogTermsTableVerticalHeader) { // Soll: 5 Zeichen jeweils mit `\n` (last `\n` removed) DEBUG_LOG(m_logger) << "Data: \"" << qbaLogTermsTableVerticalHeader << "\""; DEBUG_LOG(m_logger) << "Data.size: Soll/Ist: 5 + 4 = 9/" << qbaLogTermsTableVerticalHeader.size(); if (qbaLogTermsTableVerticalHeader.size() == 9) return true; else { ERROR_LOG(m_logger) << "Load csv data 'Logistical Terms Table Vertical Header' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Logistical Terms Table Vertical Header' failed!"), tr("CSV Load Error")); return false; } } // end of checkLogTermsTableVerticalHeader() //----------------------------------------------------------------------------- bool CSVImport::checkLogTermsTableMatrix(const QByteArray &qbaLogTermsTableMatrix) { // Soll: 5 Zeilen mit jeweils 5 Eintraegen (jeweils max. 2 Zeilen je max 10 Zeichen) getrennt durch `;` jeweils abgeschlossen mit `\n` (last `\n` removed) DEBUG_LOG(m_logger) << "Data: \"" << qbaLogTermsTableMatrix << "\""; // split funktioniert hier nicht wenn in einem quoted Eintrag '\n' vorkommt. -> eigene split Funktion verwenden QStringList qsl = m_logisticalTermsModel->splitQuotedStringToRows(QString::fromLatin1(qbaLogTermsTableMatrix)); DEBUG_LOG(m_logger) << "Line entry size: Soll/Ist: 5/" << qsl.size(); // check number of seperators if (qsl.size() == 5) { // Begrenzung auf 10 Zeichen per Zeile pruefen... // QStringList aus den 25 Matrix Eintraegen erstellen QString qsMatrix = qsl.join(QLatin1Char(';')); QStringList qsl2 = qsMatrix.split(QLatin1Char(';')); for (int i = 0; i < qsl2.size(); ++i) { const QString &qsEntry = qsl2.at(i); if (! qsEntry.contains(QLatin1Char('\n'))) { // non quoted if (qsEntry.size() > 10) { ERROR_LOG(m_logger) << "Load csv data 'Logistical Terms Table Matrix' failed! Maximum number of 10 characters per entry line exceeded in cell " << i + 1; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Logistical Terms Table Matrix' failed! Maximum number of 10 characters per entry line exceeded."), tr("CSV Load Error")); return false; } else continue; } else { // quoted if ((qsEntry.size() > 21) || (qsEntry.indexOf(QLatin1Char('\n')) > 10)) { ERROR_LOG(m_logger) << "Load csv data 'Logistical Terms Table Matrix' failed! Maximum number of 10 characters per entry line exceeded in cell " << i + 1; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Logistical Terms Table Matrix' failed! Maximum number of 10 characters per entry line exceeded."), tr("CSV Load Error")); return false; } } } return true; } else { ERROR_LOG(m_logger) << "Load csv data 'Logistical Terms Table Matrix' failed!"; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Logistical Terms Table Matrix' failed!"), tr("CSV Load Error")); return false; } } // end of checkLogTermsTableMatrix() //----------------------------------------------------------------------------- bool CSVImport::checkPhraseTableMatrix(const QByteArray &qbaPhraseTableMatrix) { // Soll: 13 Zeilen mit jeweils 2 Eintraegen (jeweils max. 15 Zeichen) getrennt durch `;` jeweils mit `\n` (last `\n` removed) DEBUG_LOG(m_logger) << "Data: \"" << qbaPhraseTableMatrix << "\""; const QList qbaLinesList = qbaPhraseTableMatrix.split('\n'); // check number of seperators if (qbaLinesList.size() != 13) { ERROR_LOG(m_logger) << "Load csv data 'Phrase Table Matrix' failed! Invalid Line entry size: Soll/Ist: 13/" << qbaLinesList.size(); LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Phrase Table Matrix' failed! Invalid Line entry size."), tr("CSV Load Error")); return false; } // Inhalt darf nicht leer sein da das Speichern im Model indiziert auf Basis von 'split()' erfolgt. Bei leeren Eintraegen stimmt dann die Anzahl nicht!! for (const QByteArray &line : qbaLinesList) { const QList qbaLine = line.split(';'); if (qbaLine.size() == 2) { if (qbaLine.at(0).isEmpty() || qbaLine.at(1).isEmpty() || qbaLine.at(0).size() > 15 || qbaLine.at(1).size() > 15) { ERROR_LOG(m_logger) << "Load csv data 'Phrase Table Matrix' failed! Invalid number of entrys or entry size is < 1 or >> 15. "; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Phrase Table Matrix' failed! Invalid number of entrys or entry size is less than 1 or greater 15."), tr("CSV Load Error")); return false; } else continue; } else { ERROR_LOG(m_logger) << "Load csv data 'Phrase Table Matrix' failed! Invalid number of entrys or empty entry per line. "; LtsWidgets::LtsMessageDialog::showMessage(this, tr("Load csv data 'Phrase Table Matrix' failed! Invalid number of entrys or empty entry per line."), tr("CSV Load Error")); return false; } } return true; } // end of checkPhraseTableMatrix() //----------------------------------------------------------------------------- void CSVImport::removeQuotedCharInNote(QByteArray &qbaNote) { if (qbaNote.startsWith('"')) qbaNote.remove(0, 1); if (qbaNote.endsWith('"')) qbaNote.remove(qbaNote.lastIndexOf('"'), 1); } // end of removeQuotedCharInNote() //----------------------------------------------------------------------------- bool CSVImport::removeRedundantCvsSeperators(QByteArray &qbaLine, int desiredValue) const { int seperatorCnt = 0; // check for csv seperators for (const char c : qAsConst(qbaLine)) { if (c == ';') ++seperatorCnt; } if (seperatorCnt < desiredValue) { ERROR_LOG(m_logger) << "Seperator values less than desired values."; return false; } while (seperatorCnt > desiredValue) { qbaLine.remove(qbaLine.lastIndexOf(';'), 1); --seperatorCnt; } DEBUG_LOG(m_logger) << "+++Data: \"" << qbaLine << "\"" << " Soll/Ist:: " << desiredValue << "/" << seperatorCnt; return true; } // end of removeRedundantCvsSeperators() //-----------------------------------------------------------------------------