From ae2c674b81e03909566ac82e4e580007a93e79f1 Mon Sep 17 00:00:00 2001 From: Torsten Wierschin Date: Fri, 17 Jul 2026 13:13:02 +0200 Subject: [PATCH] chore(repo): replace app symlink/submodule with in-repo app directory; enable committing app/Dockerfile and tooling\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- app | 1 - app/.dockerignore | 12 + app/.gitignore | 77 +++ app/Dockerfile | 129 +++++ app/README.md | 496 ++++++++++++++++++ app/__pycache__/run_migration.cpython-312.pyc | Bin 0 -> 5908 bytes app/migration_pipeline/__init__.py | 1 + app/migration_pipeline/api_client.py | 150 ++++++ app/migration_pipeline/build_runner.py | 185 +++++++ app/migration_pipeline/chunker.py | 183 +++++++ app/migration_pipeline/config.py | 96 ++++ app/migration_pipeline/git_manager.py | 135 +++++ app/migration_pipeline/migrator.py | 408 ++++++++++++++ app/migration_pipeline/models.py | 151 ++++++ app/migration_pipeline/orchestrator.py | 204 +++++++ app/migration_pipeline/quality_gate.py | 56 ++ app/migration_pipeline/quarantine.py | 190 +++++++ app/migration_pipeline/reporter.py | 221 ++++++++ app/migration_pipeline/test_generator.py | 156 ++++++ app/requirements.txt | 2 + app/run_docker.sh | 298 +++++++++++ app/run_migration.py | 143 +++++ app/tests/test_api_client.py | 32 ++ app/tests/test_config.py | 58 ++ app/tests/test_run_migration.py | 34 ++ 25 files changed, 3417 insertions(+), 1 deletion(-) delete mode 160000 app create mode 100644 app/.dockerignore create mode 100644 app/.gitignore create mode 100644 app/Dockerfile create mode 100644 app/README.md create mode 100644 app/__pycache__/run_migration.cpython-312.pyc create mode 100644 app/migration_pipeline/__init__.py create mode 100644 app/migration_pipeline/api_client.py create mode 100644 app/migration_pipeline/build_runner.py create mode 100644 app/migration_pipeline/chunker.py create mode 100644 app/migration_pipeline/config.py create mode 100644 app/migration_pipeline/git_manager.py create mode 100644 app/migration_pipeline/migrator.py create mode 100644 app/migration_pipeline/models.py create mode 100644 app/migration_pipeline/orchestrator.py create mode 100644 app/migration_pipeline/quality_gate.py create mode 100644 app/migration_pipeline/quarantine.py create mode 100644 app/migration_pipeline/reporter.py create mode 100644 app/migration_pipeline/test_generator.py create mode 100644 app/requirements.txt create mode 100755 app/run_docker.sh create mode 100644 app/run_migration.py create mode 100644 app/tests/test_api_client.py create mode 100644 app/tests/test_config.py create mode 100644 app/tests/test_run_migration.py diff --git a/app b/app deleted file mode 160000 index 0b52799a2..000000000 --- a/app +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 0b52799a26e26dc708f9042bcd92ffb346bd2036 diff --git a/app/.dockerignore b/app/.dockerignore new file mode 100644 index 000000000..3f648fa55 --- /dev/null +++ b/app/.dockerignore @@ -0,0 +1,12 @@ +.git +.gitignore +__pycache__/ +*.pyc +*.pyo +*.pyd +.venv/ +venv/ +build/ +dist/ +node_modules/ +.cache/ diff --git a/app/.gitignore b/app/.gitignore new file mode 100644 index 000000000..41ee3de11 --- /dev/null +++ b/app/.gitignore @@ -0,0 +1,77 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Linker files +*.ilk + +# Debugger Files +*.pdb + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll +*.so.* + + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# Build directories +build/ +Build/ +build-*/ + +# CMake generated files +CMakeFiles/ +CMakeCache.txt +cmake_install.cmake +Makefile +install_manifest.txt +compile_commands.json + +# Temporary files +*.tmp +*.log +*.bak +*.swp + +# vcpkg +vcpkg_installed/ + +# debug information files +*.dwo + +# test output & cache +Testing/ +.cache/ +.env + +1q.sh +q1_files.txt +quarantine +reports +workspace +migration_pipeline/__pycache__/ diff --git a/app/Dockerfile b/app/Dockerfile new file mode 100644 index 000000000..4cda435ce --- /dev/null +++ b/app/Dockerfile @@ -0,0 +1,129 @@ +FROM ubuntu:24.04 + +ENV DEBIAN_FRONTEND=noninteractive \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + QT_VERSION=6.8.3 \ + QT_ARCH=linux_gcc_64 \ + QT_BASE_DIR=/opt/Qt \ + QT_ROOT=/opt/Qt/6.8.3/gcc_64 \ + Qt6_ROOT=/opt/Qt/6.8.3/gcc_64 \ + CMAKE_PREFIX_PATH=/usr/local:/opt/Qt/6.8.3/gcc_64 \ + PATH=/opt/Qt/6.8.3/gcc_64/bin:$PATH + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + sudo \ + git \ + cmake \ + ninja-build \ + clang \ + build-essential \ + gnupg \ + curl \ + jq \ + openssh-client \ + pkg-config \ + libgl1-mesa-dev \ + libxkbcommon-x11-0 \ + libfontconfig1 \ + libdbus-1-3 \ + libpulse0 \ + libxcb-glx0 \ + libxcb-cursor0 \ + libpq-dev \ + libbz2-dev \ + libpoppler-dev \ + libpoppler-qt6-dev \ + python3 \ + python3-venv \ + python3-pip \ + && ln -sf /usr/bin/python3 /usr/local/bin/python3 \ + && ln -sf /usr/bin/python3 /usr/local/bin/python \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt + +RUN python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --upgrade pip setuptools wheel \ + && /opt/venv/bin/pip install -r /app/requirements.txt aqtinstall \ + && mkdir -p "${QT_BASE_DIR}" \ + && /opt/venv/bin/python -m aqt install-qt linux desktop "${QT_VERSION}" "${QT_ARCH}" -m qtmultimedia qtserialport qtimageformats qtwebsockets qtwebengine qtpdf qtwebview --outputdir "${QT_BASE_DIR}" \ + && test -f "${QT_ROOT}/lib/cmake/Qt6/Qt6Config.cmake" + +# Build and install Log4Qt from source (provides headers + shared libs) +RUN set -eux; \ + git clone --depth 1 https://github.com/MEONMedical/Log4Qt.git /tmp/log4qt; \ + mkdir -p /tmp/log4qt/build; \ + cd /tmp/log4qt/build; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_EXTENSIONS=OFF -DCMAKE_PREFIX_PATH="${QT_ROOT}" -DCMAKE_INSTALL_PREFIX=/usr/local ..; \ + cmake --build . -- -j$(nproc); \ + cmake --install .; \ + # create a minimal CMake config file so find_package(Log4Qt) finds the installed library + mkdir -p /usr/local/lib/cmake/Log4Qt; \ + libpath="$((ls /usr/local/lib/libLog4Qt.so* 2>/dev/null || ls /usr/local/lib/liblog4qt.so* 2>/dev/null) | head -n1 || true)"; \ + if [ -n "$libpath" ]; then \ + printf '%s\n' \ + '# Minimal Log4QtConfig.cmake created by Dockerfile' \ + 'if (NOT TARGET Log4Qt::log4qt)' \ + ' add_library(Log4Qt::log4qt UNKNOWN IMPORTED)' \ + ' set_target_properties(Log4Qt::log4qt PROPERTIES' \ + " IMPORTED_LOCATION \"${libpath}\"" \ + ' INTERFACE_INCLUDE_DIRECTORIES "/usr/local/include"' \ + ' )' \ + 'endif()' \ + 'set(Log4Qt_FOUND TRUE CACHE INTERNAL "Log4Qt found by package config")' \ + 'set(Log4Qt_INCLUDE_DIRS "/usr/local/include" CACHE INTERNAL "Log4Qt include dir")' \ + "set(Log4Qt_LIBRARIES \"${libpath}\" CACHE INTERNAL \"Log4Qt library\")" \ + > /usr/local/lib/cmake/Log4Qt/Log4QtConfig.cmake; \ + fi; \ + rm -rf /tmp/log4qt + +# Install additional external dependencies required by Lts modules +# Attempt to install system packages first; then build Grantlee and KDReports from source if not available +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libgeographic-dev \ + libssl-dev \ + libxml2-dev \ + libicu-dev \ + libqt6svg6-dev \ + libqt6webkit6-dev || true; \ + # Try to install Grantlee / KDReports from apt if available; ignore failures + apt-get install -y --no-install-recommends libgrantlee-qt5-dev libgrantlee-templates5-dev libkdreports-dev || true; \ + # Build Grantlee from source (KDE frameworks) if cmake config not available + if ! ldconfig -p | grep -q grantlee || ! pkg-config --exists grantlee-templates; then \ + echo "Building Grantlee from source"; \ + git clone --depth 1 https://invent.kde.org/frameworks/grantlee.git /tmp/grantlee || true; \ + if [ -d /tmp/grantlee ]; then \ + mkdir -p /tmp/grantlee/build; cd /tmp/grantlee/build; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_PREFIX_PATH="${QT_ROOT}:/usr/local" ..; \ + cmake --build . -- -j$(nproc) || true; \ + cmake --install . || true; \ + fi; \ + fi; \ + # Build KDReports from source if not available + if ! ldconfig -p | grep -q kdreports && [ ! -f /usr/local/lib/libKDReports.so ]; then \ + echo "Building KDReports from source"; \ + git clone --depth 1 https://invent.kde.org/sdk/kdreports/kdreports.git /tmp/kdreports || true; \ + if [ -d /tmp/kdreports ]; then \ + mkdir -p /tmp/kdreports/build; cd /tmp/kdreports/build; \ + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_PREFIX_PATH="${QT_ROOT}:/usr/local" ..; \ + cmake --build . -- -j$(nproc) || true; \ + cmake --install . || true; \ + fi; \ + fi; \ + # geographiclib already installed via apt; RTI (Proprietary) cannot be installed automatically + echo "Note: RTI Connext (RTI) is proprietary and must be installed manually into /opt/RTI if required." + +COPY . /app + +CMD ["python3.11", "run_migration.py", "--help"] diff --git a/app/README.md b/app/README.md new file mode 100644 index 000000000..337bc7884 --- /dev/null +++ b/app/README.md @@ -0,0 +1,496 @@ +# Qt5 → Qt6 Migrationspipeline + +Asynchrone, agentische Pipeline zur automatisierten Migration einer C++/Qt5-Codebasis nach Qt6. +Entwickelt für den Pilotbatch **Q1** (~32 k LOC, 115 Files) als Vorstufe zur vollständigen Q2-Migration (~1,2 Mio LOC, ~6 500 Files). + +--- + +## Inhaltsverzeichnis + +1. [Überblick](#überblick) +2. [Architektur](#architektur) +3. [Dual-Track-Strategie](#dual-track-strategie) +4. [Semantisches Chunking](#semantisches-chunking) +5. [Multi-Turn-Konversation](#multi-turn-konversation) +6. [Quality Gates](#quality-gates) +7. [Quarantäne & Tickets](#quarantäne--tickets) +8. [Voraussetzungen](#voraussetzungen) +9. [Einrichtung](#einrichtung) +10. [Docker](#docker) +11. [Ausführung](#ausführung) +12. [Ausgaben](#ausgaben) +13. [Konfigurationsreferenz](#konfigurationsreferenz) +14. [Modulübersicht](#modulübersicht) +15. [Burn-Phase & Skalierung auf Q2](#burn-phase--skalierung-auf-q2) +16. [FAQ](#faq) + +--- + +## Überblick + +Die Pipeline übernimmt folgende Aufgaben vollautomatisch: + +| Schritt | Beschreibung | +|---|---| +| **Batch-Isolation** | Pro Batch + Track ein eigener Git-Worktree und Build-Verzeichnis | +| **LLM-Migration** | Qt5-APIs werden per LLM (Scaleway) auf Qt6 portiert | +| **Chunk-Verarbeitung** | Dateien > 500 Zeilen werden semantisch gesplittet und parallel migriert | +| **Syntax-Check** | Schneller Per-File-Check via `compile_commands.json` ohne Full-Rebuild | +| **Test-Generierung** | QTest (Qt-naher Code) oder gtest + Smoke-Test pro Datei | +| **Quality Gates** | Build ✓ · Bestehende Tests ✓ · Neue Tests ✓ | +| **Quarantäne** | Dateien, die Gates nicht bestehen, werden isoliert + Ticket erzeugt | +| **Vergleichsreport** | Track A vs. Track B als Markdown + JSON | + +--- + +## Architektur + +``` +run_migration.py ← CLI-Einstiegspunkt +│ +└── migration_pipeline/ + ├── orchestrator.py ← Batch-Steuerung, beide Tracks parallel + │ ├── git_manager.py ← Worktree + Branch pro Batch×Track + │ ├── build_runner.py ← cmake configure/build/ctest (isoliert) + │ └── migrator.py ← Per-File Agentenloop + │ ├── chunker.py ← Semantisches C++ Chunking + │ ├── api_client.py ← Scaleway API + │ ├── test_generator.py ← QTest / gtest / Smoke + │ ├── quality_gate.py ← Gate-Auswertung + │ └── quarantine.py ← Quarantäne + Ticket + ├── reporter.py ← Vergleichsreport A vs B + ├── models.py ← Dataclasses & Enums + └── config.py ← Alle Konstanten +``` + +### Datenfluss pro Datei + +``` +Quelldatei + │ + ▼ split_semantic() wenn > 500 Zeilen +[Chunk 1] [Chunk 2] … [Chunk N] ← parallel per LLM migriert + │ + ▼ Consistency-Pass (Merge-Check via LLM) +Migrierter Code + │ + ▼ syntax_check_file() (compile_commands.json, kein Full-Rebuild) + │ + ▼ generate_tests() (QTest oder gtest + Smoke) + │ + ▼ cmake_configure() + cmake_build() + ctest + │ + ┌──┴──────────────────┐ + │ Quality Gates │ + │ ✓ Build │ + │ ✓ Bestehende Tests │ + │ ✓ Neue Tests │ + └──┬──────────────────-┘ + │ └── FAIL → Retry (max 3×) + │ └── Quarantäne + ▼ + git commit (Branch migration//) +``` + +--- + +## Dual-Track-Strategie + +Beide Tracks laufen **gleichzeitig** auf dem selben Batch. + +| | Track A – Konservativ | Track B – Modern | +|---|---|---| +| **Ziel** | Qt5→Qt6 Kompatibilität, minimale Änderungen | Qt6 + C++20 Modernisierung | +| **Stil** | Original-Stil beibehalten | Ranges, Concepts, `std::optional`, Lambdas, `[[nodiscard]]` usw. | +| **Git-Branch** | `migration//track_a_conservative` | `migration//track_b_modern` | +| **Risiko** | Niedrig | Mittel | + +Nach Abschluss des Pilotbatches folgt die **Burn-Phase**: beide Tracks werden identischen Gates unterzogen und anhand von Erfolgsquote, Warnungsanzahl und Wartbarkeit verglichen. Der Vergleichsreport enthält eine automatische Empfehlung. + +--- + +## Semantisches Chunking + +Große Dateien (> 500 Zeilen) werden **nicht** an willkürlichen Zeilengrenzen getrennt, sondern an: + +1. Klassen-/Struct-/Namespace-Definitionen (`class Foo {`, `namespace Bar {`) +2. Funktionsdefinitionen (Top-Level und Methoden) +3. Fallback: nächste Leerzeile unterhalb der 500-Zeilen-Marke + +Alle Chunks einer Datei werden **parallel** per API migriert. Anschließend läuft ein **Consistency-Pass** (separater LLM-Aufruf), der: +- doppelte `#pragma once` / Include-Guards entfernt, +- offene Namespaces schließt, +- doppelte Deklarationen bereinigt. + +--- + +## Multi-Turn-Konversation + +Die Retry-Schleife des Migrations-Agenten verwendet eine **wachsende Gesprächshistorie**, sodass das Modell beim nächsten Versuch seine eigene fehlerhafte Ausgabe zusammen mit der Compiler-Fehlermeldung sieht – analog zum Verhalten des Copilot CLI. + +### Konversationsstruktur + +``` +Iteration 0: [system, user(„Portiere folgenden Qt5-Code…")] + → reply_v1 + +Iteration 1: [system, user(„Portiere…"), assistant(reply_v1), + user(„Fehler im vorherigen Versuch:\n\nBitte korrigiere:\n")] + → reply_v2 +``` + +### Drei unabhängige Konversations-Scopes + +| Scope | Lebensdauer | Zweck | +|---|---|---| +| **Whole-file retry** (`whole_messages`) | Pro Datei; wächst über alle Retries | Modell lernt iterativ aus Compiler-Fehlern seiner eigenen Ausgabe | +| **Chunk-Migration** (je Chunk) | Pro Chunk, frisch + unabhängig | Parallele Chunk-Verarbeitung ohne gegenseitige Beeinflussung | +| **Consistency-Pass** (frisch) | Pro Datei, einmalig nach Chunk-Merge | Isoliert vom Retry-Loop; prüft nur Merge-Konsistenz | + +### Per-File-Scoping + +`whole_messages` ist eine **lokale Variable** innerhalb von `migrate_file` und wird nach Rückgabe der Funktion verworfen. Gesprächshistorie wird **niemals** zwischen Dateien oder Batches geteilt. + +### Layered API in `api_client.py` + +| Funktion | Typ | Verwendung | +|---|---|---| +| `_send_messages(client, model, messages)` | privat | Einziger HTTP-Aufruf; nimmt eine fertige Message-Liste entgegen | +| `call_api(client, model, system, user)` | öffentlich, stateless | Baut `[system, user]` und delegiert – rückwärtskompatibel | +| `call_api_multiturn(client, model, messages)` | öffentlich, multi-turn | Übergibt die komplette Verlaufsliste direkt an `_send_messages` | + +`test_generator.py` verwendet weiterhin das stateless `call_api`; nur `migrator.py` nutzt `call_api_multiturn`. + +--- + +## Quality Gates + +| Gate | Bedingung | Blockierend? | +|---|---|---| +| **Build** | `cmake --build` returncode 0 | ✅ Ja | +| **Bestehende Tests** | Vollständiger CTest-Lauf grün | ✅ Ja | +| **Neue Tests** | Generierte Unit- + Smoke-Tests grün | ✅ Ja | +| **Warnungen** | Warnungsanzahl = 0 | ⚠️ Advisory (nicht blockierend) | + +Warnungen werden vollständig geloggt und im Vergleichsreport ausgewiesen. Sie blockieren die Pipeline **nicht** (konfigurierbar über `WARNINGS_ARE_BLOCKING=True` in `config.py`). + +### Fast Syntax-Check + +Vor dem teuren Full-Build wird jede Datei einzeln per `compile_commands.json` (erzeugt bei `cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON`) syntaxgeprüft: + +``` +clang++ -fsyntax-only +``` + +Das spart bei einer 1,2-Mio-LOC-Codebasis erheblich Zeit. + +--- + +## Quarantäne & Tickets + +Dateien werden in Quarantäne verschoben, wenn: +- **3 Retry-Versuche** erschöpft sind, oder +- das **Zeitbudget** überschritten wurde (10 min für kleine Dateien, 25 min für Dateien > 1 000 Zeilen). + +Quarantäne ist **nicht-blockierend**: der Rest des Batches läuft weiter. + +### Generierte Artefakte + +Für jede quarantänisierte Datei werden zwei Dateien in `quarantine/` erzeugt: + +**`__.json`** – maschinenlesbar: +```json +{ + "file_path": "src/core/MyClass.cpp", + "track": "track_a_conservative", + "failure_class": "compile_error", + "last_error": "error: 'QRegExp' was not declared ...", + "affected_apis": ["QRegExp"], + "chunk_info": "3 chunks, ~500 Zeilen/Chunk", + "manual_recommendation": "Compile-Fehler. Prüfe Qt6-API-Änderungen für: QRegExp.", + "timestamp": "2026-07-08T18:32:12Z", + "retry_count": 3, + "elapsed_seconds": 187.4 +} +``` + +**`__.md`** – manuelles Review-Ticket: +```markdown +# Quarantäne-Ticket: MyClass.cpp +| Feld | Wert | +|---|---| +| Fehlerklasse | compile_error | +| Empfehlung | Qt6 Porting Guide: QRegExp … | +… +``` + +### Fehlerklassen + +| Klasse | Bedeutung | +|---|---| +| `compile_error` | Compilerfehler (Qt5-API entfernt / geändert) | +| `link_error` | Linkfehler (fehlende Qt6-Modulabhängigkeit) | +| `test_failure` | Semantische Abweichung in Tests | +| `test_build_failure` | Generierter Test kompiliert nicht | +| `budget_exceeded` | Zeitbudget überschritten | +| `api_error` | Scaleway-API nicht erreichbar | +| `chunk_merge_error` | Inkonsistenz nach Chunk-Merge | + +--- + +## Voraussetzungen + +### System +- Python ≥ 3.11 +- cmake ≥ 3.22 +- clang++ (für `syntax_check_file`-Fallback; optional wenn `compile_commands.json` vorhanden) +- git ≥ 2.35 (für `git worktree`) +- GTest und Qt6 installiert und für cmake auffindbar + +### Scaleway API +- Zugang zur [Scaleway AI API](https://www.scaleway.com/en/docs/ai-data/managed-inference/) +- Modelle: `glm-5.2` (große Dateien) · `qwen3.6-35b-a3b` (kleine Dateien) + +--- + +## Einrichtung + +```bash +# 1. Abhängigkeiten installieren +pip install -r requirements.txt + +# 2. API-Key setzen +echo "SCW_SECRET_KEY=your_key_here" > .env + +# 3. Prioritätsliste erstellen (eine Datei pro Zeile, in Bearbeitungsreihenfolge) +cat > q1_files.txt << 'EOF' +# cmake-Dateien zuerst (beeinflussen Build aller anderen) +src/CMakeLists.txt +src/core/CMakeLists.txt +# Header-Dateien +src/core/MyClass.h +src/core/OtherClass.h +# Implementierungsdateien +src/core/MyClass.cpp +src/core/OtherClass.cpp +EOF +``` + +> **Dateireihenfolge**: Die Pipeline verarbeitet Dateien **exakt in der angegebenen Reihenfolge** (innerhalb des Tracks). Empfohlene Reihenfolge: cmake-Dateien → Header → kleine .cpp → große .cpp. + +--- + +## Docker + +Das mitgelieferte `Dockerfile` enthält alle System-Abhängigkeiten (Ubuntu 22.04, Python 3.11, cmake, clang, git) und eignet sich für eine reproduzierbare Ausführung ohne lokale Installation. + +### Image bauen & starten (empfohlen: `run_docker.sh`) + +```bash +# Build + interaktive Shell (Host-UID/GID werden automatisch gemappt) +./run_docker.sh + +# Build + direkte Ausführung der Pipeline +./run_docker.sh python3.11 run_migration.py \ + --batch-id q1_pilot \ + --file-list ./q1_files.txt +``` + +Das Skript: +- baut das Image unter dem Namen `migration-pipeline:ubuntu22.04`, +- mountet das aktuelle Verzeichnis als `/app` (Änderungen wirken sofort), +- mappt die Host-UID/GID, sodass erstellte Dateien dem aufrufenden Benutzer gehören, +- vergibt bei Bedarf `sudo`-Rechte im Container. + +### Manuelles `docker build` / `docker run` + +```bash +# Image bauen +docker build -t migration-pipeline:ubuntu22.04 . + +# API-Key übergeben und Pipeline starten +docker run --rm -it \ + -v "$(pwd):/app" \ + -w /app \ + -e SCW_SECRET_KEY="your_key_here" \ + migration-pipeline:ubuntu22.04 \ + python3.11 run_migration.py \ + --batch-id q1_pilot \ + --file-list ./q1_files.txt + +# Dry-Run (kein API-Aufruf, kein Key erforderlich) +docker run --rm -it \ + -v "$(pwd):/app" \ + -w /app \ + migration-pipeline:ubuntu22.04 \ + python3.11 run_migration.py \ + --batch-id q1_pilot \ + --file-list ./q1_files.txt \ + --dry-run +``` + +### Hinweise + +| Thema | Empfehlung | +|---|---| +| **Scaleway-Env** | Benötigt: `SCW_ACCESS_KEY`, `SCW_SECRET_KEY`, `SCW_DEFAULT_ORGANIZATION_ID`, `SCW_DEFAULT_PROJECT_ID`, `SCW_SERVERLESS` (runtime env oder `.env`) | +| **Qt6 / GTest** | Nicht im Image enthalten – das C++-Repository muss bereits konfigurierte cmake-Targets mitbringen oder per Bind-Mount bereitgestellt werden | +| **Image-Name** | Über die Umgebungsvariable `IMAGE_NAME` anpassbar, z.B. `IMAGE_NAME=my-registry/migration-pipeline:1.0 ./run_docker.sh` | + +--- + +## Ausführung + +### Dry-Run (kein API-Aufruf) +```bash +python run_migration.py \ + --batch-id q1_pilot \ + --file-list ./q1_files.txt \ + --dry-run +``` +Gibt Dateiliste mit Zeilenanzahl und Chunk-Schätzung aus. + +### Pilotbatch Q1 +```bash +python run_migration.py \ + --batch-id q1_pilot \ + --file-list ./q1_files.txt \ + --model glm \ + --repo-root . \ + --source-root ./src +``` + +### Mit Debug-Ausgabe +```bash +python run_migration.py --batch-id q1_pilot --file-list ./q1_files.txt -v +``` + +### Alle Optionen +``` +--batch-id Eindeutiger Name für diesen Batch (z.B. q1_pilot) +--file-list Pfad zur priorisierten Dateiliste +--repo-root Wurzel des git-Repositories (Standard: .) +--source-root Wurzel des C++ Quellbaums (Standard: .) +--model Modellfamilie für API-Aufrufe: glm|qwen (Standard: glm) +--dry-run Nur Dateiliste und Chunk-Schätzung ausgeben +--verbose, -v Debug-Logging aktivieren +``` + +--- + +## Ausgaben + +``` +workspace/ + q1_pilot/ + track_a_conservative/ ← git worktree, Branch migration/q1_pilot/track_a_conservative + src/... ← migrierte Quelldateien (nach jedem Erfolg committed) + tests_generated/ ← generierte Unit- und Smoke-Tests + build/ ← isoliertes cmake Build-Verzeichnis + track_b_modern/ ← analog für Track B + +quarantine/ + MyClass_track_a_conservative_20260708T183200Z.json + MyClass_track_a_conservative_20260708T183200Z.md ← Review-Ticket + +reports/ + q1_pilot_comparison.md ← Track A vs B Vergleich (Markdown) + q1_pilot_comparison.json ← Track A vs B Vergleich (maschinenlesbar) +``` + +### Beispiel-Konsolenausgabe +``` +================================================================ + Batch: q1_pilot +================================================================ + Track A (konservativ): 108/115 OK | 7 quarantined | 23 warnings | avg 142s + Track B (modern) : 97/115 OK | 18 quarantined | 41 warnings | avg 198s +================================================================ +``` + +--- + +## Konfigurationsreferenz + +Alle Werte in `migration_pipeline/config.py`: + +| Variable | Standard | Beschreibung | +|---|---|---| +| `MODEL_GLM` | `glm-5.2` | Modell-ID der GLM-Familie | +| `MODEL_QWEN` | `qwen3.5-397b-a17b` | Modell-ID der QWEN-Familie | +| `DEFAULT_MODEL_FAMILY` | `glm` | Standard-Auswahl für `--model` | +| `LARGE_FILE_THRESHOLD` | `1000` | Zeilengrenze für Time-Budget (klein/groß) | +| `MAX_CONCURRENT_TASKS` | `3` | Parallele Worker pro Track | +| `CHUNK_SIZE_LINES` | `500` | Max. Zeilen pro Chunk | +| `SOFT_BUDGET_PER_FILE` | `600` s | Soft-Budget (Warnung, kein Abbruch) | +| `HARD_BUDGET_SMALL_FILE` | `600` s | Hard-Budget für kleine Dateien | +| `HARD_BUDGET_LARGE_FILE` | `1500` s | Hard-Budget für große Dateien | +| `MAX_RETRIES` | `3` | Max. Versuche pro Datei und Track | +| `WARNINGS_ARE_BLOCKING` | `False` | `True` = Warnungen blockieren Gate | +| `WORKSPACE_ROOT` | `workspace/` | Verzeichnis für git-Worktrees | +| `QUARANTINE_DIR` | `quarantine/` | Quarantäne-Artefakte | +| `REPORTS_DIR` | `reports/` | Vergleichsreports | + +--- + +## Modulübersicht + +| Modul | Verantwortung | +|---|---| +| `config.py` | Zentralkonfiguration, Konstanten, Umgebungsvariablen | +| `models.py` | Dataclasses (`FileResult`, `BatchResult`, `QuarantineEntry` usw.), Enums | +| `chunker.py` | Semantisches C++ Chunking an Klassen-/Funktionsgrenzen | +| `api_client.py` | Scaleway API Client (async, `httpx`); layered API: stateless `call_api` + multi-turn `call_api_multiturn` | +| `build_runner.py` | cmake configure/build, CTest, `compile_commands.json`-Parser | +| `test_generator.py` | QTest/gtest/Smoke-Generierung, idempotente CMakeLists-Registrierung | +| `quality_gate.py` | Aggregierte Gate-Auswertung (Build · bestehende Tests · neue Tests) | +| `quarantine.py` | Fehlerklassifikation, Qt5-API-Erkennung, JSON+Markdown-Ticket | +| `git_manager.py` | `git worktree` Lifecycle, Per-File-Commits | +| `migrator.py` | Per-File Agentenloop: Chunk-Strategie, Multi-Turn-Retry-Konversation, Budget-Kontrolle | +| `orchestrator.py` | Batch-Orchestrierung, Dual-Track parallel, cmake-Konfiguration | +| `reporter.py` | Track A vs B Vergleichsreport (Markdown + JSON) | +| `run_migration.py` | CLI, Argument-Parsing, Dry-Run | + +--- + +## Burn-Phase & Skalierung auf Q2 + +### Burn-Phase (nach Q1) +1. Beide Branches (`track_a_conservative`, `track_b_modern`) auf identischer Testinfrastruktur deployen. +2. Stabilität, Warnungsfreiheit und Wartbarkeit vergleichen (Report `q1_pilot_comparison.md`). +3. Gewinner-Track als Basis für Q2 wählen (oder hybride Strategie: Track A für risikoarme Module, Track B für gut getestete Module). + +### Skalierung auf Q2 +| Parameter | Q1 (Pilot) | Q2 (Vollmigration) | +|---|---|---| +| Dateien | ~115 | ~6 500 | +| LOC | ~32 000 | ~1 200 000 | +| Empfohlene Worker | 3 | 5–8 (anpassen je API-Rate-Limit) | +| Batches | 1 | ~57 (à 115 Files) | +| Erwartete Laufzeit* | ~4–6 h | ~200–300 h gesamt | + +> *Schätzung bei 3 Workern, avg. 150 s/Datei, 80 % Erfolgsquote. + +Empfohlene Batch-Größe für Q2: **100–150 Files** (ca. 30 k LOC), sortiert nach Abhängigkeitstiefe (cmake → Header → Implementierung). + +--- + +## FAQ + +**Warum git worktrees statt direktem Arbeiten im Repository?** +Damit Track A und Track B niemals denselben Dateisystem- oder Build-Zustand teilen. Async-Worker würden sich sonst gegenseitig überschreiben. + +**Was passiert mit quarantänierten Dateien nach dem Batch?** +Sie werden wiederhergestellt (Original bleibt erhalten), das Ticket landet in `quarantine/`. Die Datei ist mit Fehlerklasse, betroffenen APIs und Empfehlung indiziert und kann gezielt manuell migriert werden. + +**Kann ich nur Track A oder nur Track B laufen lassen?** +Aktuell nicht über CLI-Flag. Einfachste Lösung: `TRACK_B_ID` in `orchestrator.py` auskommentieren. Ein `--track` CLI-Parameter ist als spätere Erweiterung vorgesehen. + +**Was tun bei Rate-Limiting der Scaleway API?** +`MAX_CONCURRENT_TASKS` in `config.py` reduzieren. Alternativ Exponential-Backoff in `api_client.py` ergänzen. + +**Sind die generierten Tests vollständig korrekt?** +Nein – sie sind ein Ausgangspunkt. Die Quality Gates stellen sicher, dass sie *kompilieren und laufen*, aber keine logischen Fehler enthalten. Manuelle Nachkontrolle empfohlen, besonders für fachkritische Module. + +**Wie wird mit `.cmake`-Dateien umgegangen?** +`.cmake`-Dateien werden wie C++ Dateien per LLM migriert (cmake-spezifische Qt5→Qt6 Syntax). Da sie nicht kompiliert werden, entfällt der Syntax-Check; die Gates Build/Test gelten nach dem nächsten Full-Build. diff --git a/app/__pycache__/run_migration.cpython-312.pyc b/app/__pycache__/run_migration.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e0e5abb4998bd2b6d7ad194b3eb77099c83bdc87 GIT binary patch literal 5908 zcmb_gTTC3+8J^jlz0O{_8DkfZF~n@b;)@a+6YRz|S6{%!b}EtBlVN9oA-gl1Gqbi? zFA|M%ff~u{wh}~66so>BvZc6ErKV~hQmIla^#uyiuBSwblJp^OBshKWQ~z^zXLe1% zZKWQ;IcLuK|Ns1R=A7^Uzx|uv@1>wHSI&>`3{uor_`xi;LWf>|6FPI0NDWaUE!yIA z!Zu{1ao--d57|k}3^Am23^_>a9CAX-#9ayZkQ-!K(eV~Dl1^b2U@=r5WTNAQxtOBD9L}X=m||t*eC3fP-=#zsFP>yN4T0DFrhJ zKr625+_0d>#yBuCtmgcGfj2e#4t`nDf z45qKYd%$3a1x=2fRFx5VG-5McS0yy8Y7)YpYxwm!?r~`2kuoay7|}=}i7%Q(a7C#) zto1BZQ`EyBHO}_jI{UM;_xFC>@=?oF*DAy2m~BhUwtS#6AMDA8>i*?o9L{x+reftA z?SldGNjhK@?l!*HRDzx~@3k#qpS07I?!lV4WT*^%uV`VC(b>W%MTv~=H~SgpUh%Ws zp+rZ98V{H=r50wKA2>`oSfOOeCA#l0qDS|X=+ZoGOqDzbG+1==%Z~b`IvDOe9SnDK8A^JW$!cF2G+Kf zte8Eq;(n#IqPuL*+~eC#nNn-6ZE1LHRK}KZXYA&CO?rRmmA=EY06uK8_olUi?-@^F zf6ILFS`r&}eD>E&*_SfCgKJe4sZJj~hX@b@qHzS^F!~hHNYdb{!S&EANQwz@faFm? ztO@S;xxQ9}4;8>hKugADo!6vu-f?KZ;Wz{6ucd9r4jImq7#_9s;1kIIe|h)@pZc_? zf8Zk5J#eYN({Re8ii#w72d9LPiU1b_Mx7NfATku=5`k0q6BbAY&prxXd1rI zI2aZS!UxfC^AI+8z&NxY9{0e%whMM`iuxyWAe>R*wn001{->;lM|KZE;y?kfK!iB}! z$lZ#&T@M=Wwm%sBqy3ZaKd_&kSUlXb80h`X-M1FTK0d0T9NxEkPzW~a3EXzU0Kx5E z*gXiP^f*3y8;W^9CvBtDq#ba$7zWG5Oqp>UoHrD64zN=J++;G$I9_){F{LIQ83zhy z9OG83C<%(Ri71Gk-CK@W=J=#DV;{F7NJ&%2p^lV{bKF99bBv>q*(5_{>=&qrtsmh45b+o;Bx!0K0(2uhqb8^f2iy@k3D-s!kgj2DD_$)j znUoX}J%fcuIaP)+Gl#ASH!^(1!*K{iVn3}gV3;RNHUINVnxTr+45wZN2yceebh zpn4b}RU4k3g=&h*vo$%keu=GLX1S@(ygxYAoe$RLf(=W-hUH*mF1T+gxNkXl;5*9U z7@()^d0+5W-%Q`l)6?|BU?_Wh?#%5o3zaKXO>l_u$q(P`fs^kJ-15)(e;Ioq{4VjE z#BbI4=NERr*OsemS*mMUscZe~f#sUBpSkzu} zTWUZGQl{a%@Bfb)fhRPA|Aj_K43ngwcsaN9rAn*@$r?x>%Dvv?%9vLI4)nJER0F(U zi4zIhgc399Kp8WFY?c)zEe0nXGbSTg$Y~*b@Ugg{X%Ij#jTub?V=}qT#q|D5iu^`O z;$#t+m^>m&s7S+Pt;n02rPk3_iv&d=>vbdPjg*We(O|};crwkFk&N^Um@{y?N+=iv zH5JLatVtq?@r9(KSydzsM<*@`%i)j{iP+Ig@RkOnPr%r5=whoDk9thErVkHdLZyxn z!=tj!;aN@ARp5`!qBJ7FeQf7qDTLrnI1&7d;W;o3X?;7o8jFF|=%8j3V z?m4b731m@{o@zbqv1k;=6-X6$9i^mn+r~_)MC+_X>P1deIUrmdPNd8%3ij720IN}z zf}>!+NHWs4=+$&pmm&;9c1B1ES0obvQ*Hr`CItl06gZAXA#qUjbNCS%I|hYoouzZp z3tz08upKxVhKuJ#HOBLXPY^{OY!J-?Dtd`MJgDLHhddz-2Pv+Q39tZ*umq7n{V?7M z|Jt`uO;M}<>YTrE$={f-YFn-4aR$nnx}s;GF7S^HL6z-?*#t;p)8q$7?Ri zQTvDjlJmrDHO#L1!+C$_nuiEDK)_d~UY!qB5`pe51h$z1JzEF>!P=4c_iW)nW%eSO z-}A)0uOi3pTw-@Fv-OKi{SUDehfsp0ZGfF~R0&JXyFE$iRxk%tEFuDES)~--VBSJg zYkE^8HLYt$OCegMH`l*IePGiueI#WwngPgNM1G)m7EX=hTxLTN?N4}%v)~N{?b9u4 zwz)qrTC+h39-m68Ebk_OzKV{xKM|Ou!Pi};opH|Wm0BQRct;vaBW7<)zZy-6Qu{Q& z{`WV5*jGFOrG127I$Ztfon3>21B2}xra}e$#E>zfrWCQAgNw=FR32gGY4R@*{sDqO zJp7K~CLLZD(OwY2>>;TZ*Z@vvB9-Lgl3ybb{=kOEA`oZ~wuyHV$i)Q24^0<=;1~=L z5&Tn084Dn(5ju!PcWpv_R1_Nn)B`S~B6J>0xkwa~RY;(KNDShIEw*x;>WGski)x5RQQYy)KO zi=mdg7eBu8(UrxPj>W*yKcD=|sXv|i%ssH`W^cJ?+}ZFf`-O{rH}hs@)g8)K&DGzo zU#QCMKCra=KyLSurQJuCcfYve?tmN%7|z!W`|jzVo_^>K=c~8RUA}!ezhl>j$KE@Z zuc@0;ZY#N(mZh4O<(k%f?ao|n^HObdu6F-Y?f&K3gO5B8kN11K%i$zo0X&Mw@5J*F zrm*wOu&Ty!W~rwTkidS3k`X8QY6Oo9dyEeWdJR_sR4{)S_sJ;!o`v3P&`XoV^%dCl z%MlyVNLRqt62xxp2ZE_3g_u;FDi{l*Q=n*gd441XK1e)|FhZdoTn*t0gBrR7mEpqi zJ1!3w7Q`7op@W5#7smqft?)}?GRyNH!7qh)Gyv6OcSk}MQ*r4S8i7X~QM5Bqt=Vas z{+996%=h&az2_?`_zx;T{=A=4wd+nN?Od;@pndDldgzMv+P(Cy^^0^h?OfYWF}~^a T-|gEUGVZs#*X$HiM^^hcCZ$lu literal 0 HcmV?d00001 diff --git a/app/migration_pipeline/__init__.py b/app/migration_pipeline/__init__.py new file mode 100644 index 000000000..be06ff1b8 --- /dev/null +++ b/app/migration_pipeline/__init__.py @@ -0,0 +1 @@ +"""Qt5 → Qt6 Migration Pipeline – asynchronous, dual-track, chunk-aware.""" diff --git a/app/migration_pipeline/api_client.py b/app/migration_pipeline/api_client.py new file mode 100644 index 000000000..cadc2b810 --- /dev/null +++ b/app/migration_pipeline/api_client.py @@ -0,0 +1,150 @@ +"""Scaleway API client with async support and model selection.""" + +from __future__ import annotations + +import json + +import httpx + +from migration_pipeline.config import ( + API_KEY, + API_TIMEOUT, + BASE_URL, + DEFAULT_MODEL_FAMILY, + MODEL_GLM, + MODEL_QWEN, +) + +MODEL_FAMILY_GLM = "glm" +MODEL_FAMILY_QWEN = "qwen" +_SELECTED_MODEL_FAMILY = DEFAULT_MODEL_FAMILY + +_MODEL_PAYLOADS: dict[str, dict] = { + MODEL_FAMILY_GLM: { + "model": MODEL_GLM, + "max_tokens": 16384, + "temperature": 1, + "top_p": 0.95, + "presence_penalty": 0, + "stream": True, + "response_format": {"type": "text"}, + }, + MODEL_FAMILY_QWEN: { + "model": MODEL_QWEN, + "max_tokens": 16384, + "temperature": 0.6, + "top_p": 0.95, + "presence_penalty": 0, + "stream": True, + "response_format": {"type": "text"}, + }, +} + +_MODEL_TO_FAMILY = { + profile["model"]: family for family, profile in _MODEL_PAYLOADS.items() +} + + +def set_model_family(family: str) -> None: + """Set active model family for the current pipeline run.""" + normalized = family.strip().lower() + if normalized not in _MODEL_PAYLOADS: + raise ValueError(f"Unsupported model family: {family}") + global _SELECTED_MODEL_FAMILY + _SELECTED_MODEL_FAMILY = normalized + + +def get_selected_model_family() -> str: + return _SELECTED_MODEL_FAMILY + + +def select_model(line_count: int) -> str: + """Return model configured for the selected family (line_count kept for compatibility).""" + _ = line_count + return _MODEL_PAYLOADS[_SELECTED_MODEL_FAMILY]["model"] + + +async def _read_streaming_content(response: httpx.Response) -> str: + content_parts: list[str] = [] + async for raw_line in response.aiter_lines(): + line = raw_line.strip() + if not line or line == "data: [DONE]" or not line.startswith("data: "): + continue + payload = line[len("data: "):] + try: + data = json.loads(payload) + except ValueError: + continue + choices = data.get("choices", []) + if not choices: + continue + delta = choices[0].get("delta", {}) + token = delta.get("content") + if token: + content_parts.append(token) + return "".join(content_parts) + + +async def _send_messages( + client: httpx.AsyncClient, + model: str, + messages: list[dict], +) -> str: + """Core HTTP helper: POST *messages* to the chat-completions endpoint. + + Raises ``httpx.HTTPStatusError`` on non-2xx responses so the caller can + handle retries / quarantine. + """ + family = _MODEL_TO_FAMILY.get(model) + if not family: + raise ValueError(f"Unsupported model id: {model}") + model_payload = _MODEL_PAYLOADS[family] + payload = {**model_payload, "messages": messages, "model": model} + headers = { + "Authorization": "Bearer " + API_KEY, + "Content-Type": "application/json", + } + if payload.get("stream"): + async with client.stream( + "POST", + BASE_URL, + json=payload, + headers=headers, + timeout=API_TIMEOUT, + ) as response: + response.raise_for_status() + return await _read_streaming_content(response) + response = await client.post(BASE_URL, json=payload, headers=headers, timeout=API_TIMEOUT) + response.raise_for_status() + return response.json()["choices"][0]["message"]["content"] + + +async def call_api( + client: httpx.AsyncClient, + model: str, + system_prompt: str, + user_content: str, +) -> str: + """Stateless single-turn call (backward-compatible). + + Builds a minimal ``[system, user]`` message list and delegates to + :func:`_send_messages`. + """ + messages: list[dict] = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_content}, + ] + return await _send_messages(client, model, messages) + + +async def call_api_multiturn( + client: httpx.AsyncClient, + model: str, + messages: list[dict], +) -> str: + """Multi-turn call: pass the full conversation history and return the next reply. + + The caller is responsible for maintaining and extending *messages* + (appending ``{"role": "assistant", "content": reply}`` after each call). + """ + return await _send_messages(client, model, messages) diff --git a/app/migration_pipeline/build_runner.py b/app/migration_pipeline/build_runner.py new file mode 100644 index 000000000..b825d00a0 --- /dev/null +++ b/app/migration_pipeline/build_runner.py @@ -0,0 +1,185 @@ +"""Isolated cmake / ctest subprocess helpers. + +Each track gets its own worktree + build directory, so workers never +share a build state and async concurrency is safe. +""" + +from __future__ import annotations + +import asyncio +import json +import re +import subprocess +from pathlib import Path +from typing import Tuple + + +# --------------------------------------------------------------------------- +# Type alias +# --------------------------------------------------------------------------- +RunResult = Tuple[bool, str, str] # (success, stdout, stderr) + + +# --------------------------------------------------------------------------- +# Low-level sync helpers (run in thread-pool via run_in_executor) +# --------------------------------------------------------------------------- + +def _run(cmd: list[str], cwd: Path | None = None) -> Tuple[int, str, str]: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=str(cwd) if cwd else None, + ) + return result.returncode, result.stdout, result.stderr + + +# --------------------------------------------------------------------------- +# Async wrappers +# --------------------------------------------------------------------------- + +async def cmake_configure(source_dir: Path, build_dir: Path) -> RunResult: + """Run cmake configure, enabling compile_commands.json export.""" + build_dir.mkdir(parents=True, exist_ok=True) + loop = asyncio.get_running_loop() + rc, out, err = await loop.run_in_executor( + None, + lambda: _run( + [ + "cmake", + "-B", str(build_dir), + str(source_dir), + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + "-DCMAKE_BUILD_TYPE=Debug", + ] + ), + ) + return rc == 0, out, err + + +async def cmake_build(build_dir: Path, jobs: int = 4) -> RunResult: + """Full (incremental) build of the tree in *build_dir*.""" + loop = asyncio.get_running_loop() + rc, out, err = await loop.run_in_executor( + None, + lambda: _run( + ["cmake", "--build", str(build_dir), "--", f"-j{jobs}"] + ), + ) + return rc == 0, out, err + + +async def syntax_check_file(file_path: Path, build_dir: Path) -> RunResult: + """Fast per-file syntax check using the compile command from + compile_commands.json. Falls back to a plain clang++ call when the + JSON is not available. + """ + loop = asyncio.get_running_loop() + + cmd = _compile_cmd_for_file(file_path, build_dir) + if cmd is None: + # Fallback: bare syntax-only compile with C++20 + cmd = [ + "clang++", + "-std=c++20", + "-fsyntax-only", + "-w", # suppress all warnings for the fast check + str(file_path), + ] + + rc, out, err = await loop.run_in_executor(None, lambda: _run(cmd)) + return rc == 0, out, err + + +async def run_ctest( + build_dir: Path, + test_filter: str | None = None, +) -> RunResult: + """Execute CTest in *build_dir*. + + Pass *test_filter* to run only a subset (regex forwarded to ``-R``). + """ + cmd = ["ctest", "--test-dir", str(build_dir), "--output-on-failure"] + if test_filter: + cmd += ["-R", test_filter] + + loop = asyncio.get_running_loop() + rc, out, err = await loop.run_in_executor(None, lambda: _run(cmd)) + return rc == 0, out, err + + +# --------------------------------------------------------------------------- +# Warning parsing +# --------------------------------------------------------------------------- + +_WARNING_RE = re.compile(r":\s*warning\s*:", re.IGNORECASE) + + +def parse_warnings(text: str) -> list[str]: + """Return all lines that contain a compiler/linker warning.""" + return [ln.strip() for ln in text.splitlines() if _WARNING_RE.search(ln)] + + +# --------------------------------------------------------------------------- +# compile_commands.json helpers +# --------------------------------------------------------------------------- + +def _load_compile_commands(build_dir: Path) -> dict[str, dict]: + """Return {absolute_file_path: entry} from compile_commands.json.""" + cc_path = build_dir / "compile_commands.json" + if not cc_path.exists(): + return {} + try: + with cc_path.open() as fh: + entries = json.load(fh) + return {e["file"]: e for e in entries if "file" in e} + except Exception: + return {} + + +def _compile_cmd_for_file( + file_path: Path, + build_dir: Path, +) -> list[str] | None: + """Build a ``-fsyntax-only`` command for *file_path* from + ``compile_commands.json``. Returns None if no entry is found. + """ + db = _load_compile_commands(build_dir) + if not db: + return None + + # Try both the resolved absolute path and as-is string. + key = str(file_path.resolve()) + entry = db.get(key) or db.get(str(file_path)) + if entry is None: + return None + + raw_cmd: str = entry.get("command", "") + if not raw_cmd: + # Some generators use "arguments" (list) instead of "command" (string). + args: list[str] = entry.get("arguments", []) + if not args: + return None + raw_cmd = " ".join(args) + + parts = raw_cmd.split() + result: list[str] = [] + skip_next = False + + for part in parts: + if skip_next: + skip_next = False + continue + if part == "-c": + result.append("-fsyntax-only") + elif part == "-o": + skip_next = True # drop the output-file argument + else: + result.append(part) + + # Ensure the entry's source file is replaced with the migrated file path. + orig_src = entry.get("file", "") + if orig_src and result and result[-1] == orig_src: + result[-1] = str(file_path) + + return result diff --git a/app/migration_pipeline/chunker.py b/app/migration_pipeline/chunker.py new file mode 100644 index 000000000..33f9159d4 --- /dev/null +++ b/app/migration_pipeline/chunker.py @@ -0,0 +1,183 @@ +"""Semantic C++ source-file chunker. + +Splits large C++ files at meaningful boundaries (class, struct, namespace, +free function definitions) so that each chunk can be migrated independently +by the LLM without losing context at arbitrary line positions. + +Secondary fallback: nearest blank line below the 500-line mark. +""" + +from __future__ import annotations + +import re +from typing import List + +from migration_pipeline.models import CodeChunk + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def split_semantic(source: str, max_lines: int = 500) -> List[CodeChunk]: + """Split *source* into chunks of at most *max_lines* lines. + + Returns a list with a single CodeChunk (boundary_type="complete") when + the file is small enough to fit in one shot. + """ + lines = source.splitlines(keepends=True) + n = len(lines) + + if n <= max_lines: + return [CodeChunk(0, 0, n, source, "complete")] + + boundaries = _find_top_level_boundaries(lines) + return _partition(lines, boundaries, max_lines) + + +# --------------------------------------------------------------------------- +# Boundary detection +# --------------------------------------------------------------------------- + +# Matches the opening of a top-level class/struct/namespace/enum-class. +_COMPOSITE_RE = re.compile( + r"""^\s* + (?:template\s*<[^>]*>\s*)? # optional template<...> + (?:class|struct|namespace|enum\s+class)\s+\w + """, + re.VERBOSE, +) + +# Matches a free-function or method *definition* opening (has `{` at end of +# logical line, not just a forward declaration ending with `;`). +_FUNC_DEF_RE = re.compile( + r"""^ # start of line + [\w:<>\*&~\s]+ # return type / qualifier + \s+\w[\w:<>]* # function name + \s*\( # opening paren + """, + re.VERBOSE, +) + + +def _find_top_level_boundaries(lines: List[str]) -> List[int]: + """Return line indices (0-based) that start a new top-level declaration.""" + boundaries: List[int] = [0] + brace_depth = 0 + in_block_comment = False + i = 0 + + while i < len(lines): + raw = lines[i] + line = raw.strip() + + # ---- Block-comment tracking ---------------------------------------- + if in_block_comment: + if "*/" in raw: + in_block_comment = False + i += 1 + continue + + if "/*" in raw and "*/" not in raw: + in_block_comment = True + + # ---- Skip single-line comments, empty lines, preprocessor ---------- + if not line or line.startswith("//") or line.startswith("#"): + i += 1 + continue + + # ---- At top level: look for new declarations ----------------------- + if brace_depth == 0: + if _COMPOSITE_RE.match(raw) or _looks_like_func_def(lines, i): + if i > 0 and i not in boundaries: + boundaries.append(i) + + # ---- Track brace depth (naïve, ignores strings/chars) ------------- + # Remove string/char literals to avoid counting braces inside them. + cleaned = _strip_literals(raw) + brace_depth += cleaned.count("{") - cleaned.count("}") + brace_depth = max(0, brace_depth) + + i += 1 + + return sorted(boundaries) + + +def _looks_like_func_def(lines: List[str], start: int) -> bool: + """Check whether lines[start:] begins a function *definition* (has `{`).""" + # Collect up to 5 continuation lines for multi-line signatures. + snippet = "" + for j in range(start, min(start + 5, len(lines))): + snippet += lines[j] + if "{" in snippet: + return bool(_FUNC_DEF_RE.match(snippet.lstrip())) + if ";" in snippet: + return False # forward declaration + return False + + +_STRING_RE = re.compile(r'"[^"\\]*(?:\\.[^"\\]*)*"') +_CHAR_RE = re.compile(r"'[^'\\]*(?:\\.[^'\\]*)*'") + + +def _strip_literals(s: str) -> str: + s = _STRING_RE.sub('""', s) + s = _CHAR_RE.sub("''", s) + return s + + +# --------------------------------------------------------------------------- +# Partitioning +# --------------------------------------------------------------------------- + +def _partition( + lines: List[str], + boundaries: List[int], + max_lines: int, +) -> List[CodeChunk]: + """Greedily group lines into chunks that respect *boundaries* and *max_lines*.""" + chunks: List[CodeChunk] = [] + chunk_idx = 0 + i = 0 + n = len(lines) + boundary_set = set(boundaries) + + while i < n: + target_end = min(i + max_lines, n) + + # Prefer ending at the latest semantic boundary before target_end. + best_end = _best_boundary_before(boundaries, i + 1, target_end) + + if best_end is None or best_end <= i: + # No semantic boundary: fall back to the nearest blank line. + best_end = _nearest_blank_before(lines, target_end, i) + + content = "".join(lines[i:best_end]) + boundary_type = "semantic" if best_end in boundary_set else "fallback" + if i == 0 and best_end == n: + boundary_type = "complete" + + chunks.append(CodeChunk(chunk_idx, i, best_end, content, boundary_type)) + chunk_idx += 1 + i = best_end + + return chunks + + +def _best_boundary_before( + boundaries: List[int], lo: int, hi: int +) -> int | None: + """Return the largest boundary in (lo, hi], or None.""" + best = None + for b in boundaries: + if lo < b <= hi: + best = b + return best + + +def _nearest_blank_before(lines: List[str], target: int, floor: int) -> int: + """Search backwards from *target* for a blank line; return its index+1.""" + for j in range(min(target, len(lines)) - 1, max(floor, target - 60), -1): + if not lines[j].strip(): + return j + 1 + return target # no blank found → hard cut diff --git a/app/migration_pipeline/config.py b/app/migration_pipeline/config.py new file mode 100644 index 000000000..66de482d9 --- /dev/null +++ b/app/migration_pipeline/config.py @@ -0,0 +1,96 @@ +"""Central configuration for the Qt5→Qt6 migration pipeline.""" + +from __future__ import annotations + +import os +from pathlib import Path + +from dotenv import load_dotenv + +_DOTENV_LOADED = False + + +def _get_env(name: str) -> str: + """Read env var from runtime first, then .env as fallback.""" + value = os.getenv(name, "") + if value: + return value + global _DOTENV_LOADED + if not _DOTENV_LOADED: + load_dotenv(dotenv_path=Path.cwd() / ".env", override=False) + load_dotenv(override=False) + _DOTENV_LOADED = True + return os.getenv(name, "") + + +def _require_env(name: str) -> str: + value = _get_env(name) + if not value: + raise RuntimeError( + f"Missing required environment variable: {name}. " + "Set runtime env vars or provide them via .env." + ) + return value + +# --------------------------------------------------------------------------- +# Scaleway API +# --------------------------------------------------------------------------- +# Runtime credentials/config (must exist in runtime env or .env). +SCW_ACCESS_KEY: str = _require_env("SCW_ACCESS_KEY") +SCW_SECRET_KEY: str = _require_env("SCW_SECRET_KEY") +SCW_DEFAULT_ORGANIZATION_ID: str = _require_env("SCW_DEFAULT_ORGANIZATION_ID") +SCW_DEFAULT_PROJECT_ID: str = _require_env("SCW_DEFAULT_PROJECT_ID") +SCW_SERVERLESS: str = _require_env("SCW_SERVERLESS") + +API_KEY: str = SCW_SECRET_KEY +BASE_URL: str = f"https://api.scaleway.ai/{SCW_SERVERLESS}/v1/chat/completions" +API_TIMEOUT: float = 120.0 # seconds per HTTP request +API_MAX_TOKENS: int = 8192 + +# Model selection profile (CLI-selectable) +MODEL_GLM: str = "glm-5.2" +MODEL_QWEN: str = "qwen3.5-397b-a17b" +DEFAULT_MODEL_FAMILY: str = "glm" +LARGE_FILE_THRESHOLD: int = 1_000 + +# --------------------------------------------------------------------------- +# Concurrency +# --------------------------------------------------------------------------- +MAX_CONCURRENT_TASKS: int = 3 # parallel file workers per track + +# --------------------------------------------------------------------------- +# Chunking +# --------------------------------------------------------------------------- +CHUNK_SIZE_LINES: int = 500 # max lines per chunk when splitting large files + +# --------------------------------------------------------------------------- +# Time budgets (seconds) +# Soft budget → warning logged, migration continues +# Hard budget → file goes to quarantine immediately +# --------------------------------------------------------------------------- +SOFT_BUDGET_PER_FILE: float = 10 * 60 # 10 min +HARD_BUDGET_SMALL_FILE: float = 10 * 60 # 10 min (≤ LARGE_FILE_THRESHOLD lines) +HARD_BUDGET_LARGE_FILE: float = 25 * 60 # 25 min (> LARGE_FILE_THRESHOLD lines) + +# --------------------------------------------------------------------------- +# Retry / quality +# --------------------------------------------------------------------------- +MAX_RETRIES: int = 3 # per file, per track + +# Warnings: collected and reported but do NOT block quality gates. +# Set to True only if you want -Werror behaviour (future option). +WARNINGS_ARE_BLOCKING: bool = False + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- +WORKSPACE_ROOT: Path = Path("workspace") # git worktrees live here +QUARANTINE_DIR: Path = Path("quarantine") # quarantine JSON + MD tickets +REPORTS_DIR: Path = Path("reports") # batch comparison reports +TESTS_DIR_NAME: str = "tests_generated" # relative to each worktree root + +# --------------------------------------------------------------------------- +# Track identifiers +# --------------------------------------------------------------------------- +TRACK_A_ID: str = "track_a_conservative" +TRACK_B_ID: str = "track_b_modern" diff --git a/app/migration_pipeline/git_manager.py b/app/migration_pipeline/git_manager.py new file mode 100644 index 000000000..2c3ebc155 --- /dev/null +++ b/app/migration_pipeline/git_manager.py @@ -0,0 +1,135 @@ +"""Git worktree + branch management for batch isolation. + +Each batch × track combination gets its own git worktree so that: + - Track A and Track B never share filesystem state. + - Build directories are fully independent. + - Every successful file migration is committed immediately to its branch. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Low-level helper +# --------------------------------------------------------------------------- + +def _git(args: list[str], cwd: Path | None = None) -> tuple[int, str, str]: + result = subprocess.run( + ["git"] + args, + capture_output=True, + text=True, + cwd=str(cwd) if cwd else None, + ) + return result.returncode, result.stdout.strip(), result.stderr.strip() + + +# --------------------------------------------------------------------------- +# Branch / worktree lifecycle +# --------------------------------------------------------------------------- + +def branch_name(batch_id: str, track_id: str) -> str: + """Canonical branch name: ``migration//``.""" + return f"migration/{batch_id}/{track_id}" + + +def setup_worktree( + batch_id: str, + track_id: str, + repo_root: Path, + workspace_root: Path, +) -> Path: + """Ensure a git worktree exists for *batch_id* × *track_id*. + + Creates the branch (from HEAD) if it does not yet exist. + Returns the worktree path. + """ + branch = branch_name(batch_id, track_id) + worktree_path = workspace_root / batch_id / track_id + + # Create the branch if needed. + rc, _, _ = _git(["rev-parse", "--verify", branch], cwd=repo_root) + if rc != 0: + _git(["branch", branch], cwd=repo_root) + + # Remove stale worktree entry (if the directory was deleted manually). + _git(["worktree", "prune"], cwd=repo_root) + + # Check whether the worktree is already registered. + _, wt_list, _ = _git(["worktree", "list", "--porcelain"], cwd=repo_root) + if str(worktree_path) in wt_list: + return worktree_path + + worktree_path.parent.mkdir(parents=True, exist_ok=True) + rc, out, err = _git( + ["worktree", "add", str(worktree_path), branch], + cwd=repo_root, + ) + if rc != 0: + raise RuntimeError( + f"git worktree add failed for {worktree_path}: {err}" + ) + return worktree_path + + +def teardown_worktree( + batch_id: str, + track_id: str, + repo_root: Path, + workspace_root: Path, +) -> None: + """Remove the worktree (but keep the branch).""" + worktree_path = workspace_root / batch_id / track_id + _git(["worktree", "remove", "--force", str(worktree_path)], cwd=repo_root) + _git(["worktree", "prune"], cwd=repo_root) + + +# --------------------------------------------------------------------------- +# Per-file commit helpers +# --------------------------------------------------------------------------- + +def commit_migrated_file( + file_path_in_worktree: Path, + worktree_path: Path, + track_id: str, +) -> None: + """Stage and commit a single successfully migrated file.""" + _git(["add", str(file_path_in_worktree)], cwd=worktree_path) + rel = file_path_in_worktree.relative_to(worktree_path) + _git( + ["commit", "-m", f"migrate({track_id}): {rel}"], + cwd=worktree_path, + ) + + +def commit_generated_tests( + tests_dir: Path, + worktree_path: Path, + track_id: str, + test_prefix: str, +) -> None: + """Stage and commit generated test files and their CMakeLists entry.""" + _git(["add", str(tests_dir)], cwd=worktree_path) + _git( + ["commit", "-m", f"test({track_id}): generated tests for {test_prefix}"], + cwd=worktree_path, + ) + + +def commit_batch_summary( + worktree_path: Path, + track_id: str, + batch_id: str, +) -> None: + """Final commit for a batch (quarantine files, reports, etc.).""" + _git(["add", "-A"], cwd=worktree_path) + rc, _, _ = _git( + ["diff", "--cached", "--quiet"], cwd=worktree_path + ) + if rc != 0: # there are staged changes + _git( + ["commit", "-m", f"batch({track_id}): finalise {batch_id}"], + cwd=worktree_path, + ) diff --git a/app/migration_pipeline/migrator.py b/app/migration_pipeline/migrator.py new file mode 100644 index 000000000..1bc3df752 --- /dev/null +++ b/app/migration_pipeline/migrator.py @@ -0,0 +1,408 @@ +"""Per-file async migration agent. + +Lifecycle for each file x track: + 1. Read source from worktree. + 2. Split into semantic chunks when file > CHUNK_SIZE_LINES. + 3. Retry loop (up to MAX_RETRIES): + a. Migrate (chunk-aware on iteration 0; whole-file on retries). + - Whole-file path uses a multi-turn conversation: each retry appends + the error feedback and the model's previous reply to the same + message list, mirroring the Copilot CLI behaviour. + - Chunk path: each chunk migrated in parallel with its own independent + mini message list (Decision 3 - parallel execution preserved). + b. Consistency pass on iteration 0 (chunks only): uses its own fresh + multi-turn conversation so it is structurally prepared for future + per-pass retries (Decision 1 - scope B). + c. Write result to worktree. + d. Fast per-file syntax check. + e. On success: generate tests, full build + CTest, evaluate gates. + f. On all gates passed: commit, return SUCCESS. + g. On failure: collect error context, continue loop. + 4. On budget exceeded or retries exhausted: quarantine + restore original. + +Conversation history is strictly scoped per file (Decision 4 - Option A): +all message lists are local variables inside migrate_file and are discarded +when the function returns. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from pathlib import Path + +import httpx + +from migration_pipeline.api_client import call_api_multiturn, select_model +from migration_pipeline.build_runner import ( + cmake_configure, + parse_warnings, + syntax_check_file, +) +from migration_pipeline.chunker import CodeChunk, split_semantic +from migration_pipeline.config import ( + CHUNK_SIZE_LINES, + HARD_BUDGET_LARGE_FILE, + HARD_BUDGET_SMALL_FILE, + LARGE_FILE_THRESHOLD, + MAX_RETRIES, + SOFT_BUDGET_PER_FILE, +) +from migration_pipeline.git_manager import ( + commit_generated_tests, + commit_migrated_file, +) +from migration_pipeline.models import ( + FileResult, + FileStatus, + QualityGateResult, + Track, +) +from migration_pipeline.quality_gate import evaluate_gates +from migration_pipeline.quarantine import quarantine_file +from migration_pipeline.test_generator import generate_tests + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# System prompts +# --------------------------------------------------------------------------- + +_SYSTEM_A_WHOLE = ( + "Du bist ein Qt5\u2192Qt6 Migrations-Agent (Track A: konservative Kompatibilit\u00e4t). " + "Migriere ausschlie\u00dflich die notwendigen Qt5-API-Aufrufe auf ihre Qt6-\u00c4quivalente. " + "Behalte den vorhandenen Code-Stil und alle Kommentare unver\u00e4ndert. " + "Keine Modernisierungen, keine Umstrukturierungen. " + "Gib NUR den modifizierten C++ Quellcode zur\u00fcck \u2013 keine Erkl\u00e4rungen, kein Markdown." +) + +_SYSTEM_B_WHOLE = ( + "Du bist ein Qt5\u2192Qt6 Migrations-Agent (Track B: Modernisierung). " + "Migriere auf Qt6 UND modernisiere gleichzeitig auf C++20: " + "nutze auto, Ranges, Concepts, std::optional, structured bindings, " + "constexpr/consteval, [[nodiscard]], if constexpr, lambdas. " + "Ersetze veraltete Qt5-APIs vollst\u00e4ndig durch ihre Qt6-Pendants. " + "Gib NUR den modifizierten C++ Quellcode zur\u00fcck \u2013 keine Erkl\u00e4rungen, kein Markdown." +) + +_SYSTEM_A_CHUNK = ( + "Du bist ein Qt5\u2192Qt6 Migrations-Agent (Track A) f\u00fcr einen CODE-CHUNK. " + "Der Chunk ist Teil einer gr\u00f6\u00dferen Datei \u2013 erhalte alle Includes und Grenzen. " + "Migriere NUR Qt5\u2192Qt6 API-\u00c4nderungen innerhalb dieses Chunks. " + "Gib NUR C++ Quellcode zur\u00fcck." +) + +_SYSTEM_B_CHUNK = ( + "Du bist ein Qt5\u2192Qt6 Modernisierungs-Agent (Track B) f\u00fcr einen CODE-CHUNK. " + "Migriere auf Qt6 + C++20 innerhalb dieses Chunks. " + "Behalte alle Chunk-Grenzen, Namespaces und Includes exakt bei. " + "Gib NUR C++ Quellcode zur\u00fcck." +) + +_SYSTEM_MERGE_CHECK = ( + "Du pr\u00fcfst einen zusammengesetzten C++ Quellcode, der aus mehreren migrierten Chunks " + "zusammengef\u00fcgt wurde. Stelle sicher: " + "1) Include-Guards / #pragma once nur einmal vorhanden. " + "2) Alle Namespaces korrekt geschlossen. " + "3) Keine doppelten Deklarationen oder Definitionen. " + "4) Keine widerspr\u00fcchlichen Makros oder typedefs. " + "Korrigiere ausschlie\u00dflich echte Merge-Fehler \u2013 keine sonstigen \u00c4nderungen. " + "Gib NUR den korrigierten C++ Quellcode zur\u00fcck." +) + + +def _whole_system(track: Track) -> str: + return _SYSTEM_A_WHOLE if track == Track.A else _SYSTEM_B_WHOLE + + +def _chunk_system(track: Track) -> str: + return _SYSTEM_A_CHUNK if track == Track.A else _SYSTEM_B_CHUNK + + +# --------------------------------------------------------------------------- +# Multi-turn API helpers +# --------------------------------------------------------------------------- + +async def _migrate_whole_mt( + client: httpx.AsyncClient, + model: str, + code: str, + track: Track, + messages: list[dict], + error_context: str = "", +) -> str: + """Whole-file migration via a growing multi-turn conversation. + + *messages* is mutated in place: + - On the first call (empty list) the system prompt is prepended and the + initial migration request is sent. + - On subsequent calls the error feedback is appended as a new user turn + after the model's previous reply, so the model can learn from its + own mistakes within the same conversation. + + Returns the migrated code (= the latest assistant reply). + """ + if not messages: + # First call: initialise the conversation. + messages.append({"role": "system", "content": _whole_system(track)}) + user_msg = f"Portiere folgenden Qt5-Code nach Qt6:\n\n{code}" + else: + # Retry: the model sees its own previous reply + the compiler error. + user_msg = ( + f"Fehler im vorherigen Versuch:\n{error_context}\n\n" + f"Bitte korrigiere den folgenden Code:\n\n{code}" + ) + + messages.append({"role": "user", "content": user_msg}) + reply = await call_api_multiturn(client, model, messages) + messages.append({"role": "assistant", "content": reply}) + return reply + + +async def _migrate_chunk_mt( + client: httpx.AsyncClient, + model: str, + chunk: CodeChunk, + track: Track, +) -> str: + """Migrate one chunk with its own independent mini messages list. + + Each chunk is structurally multi-turn capable (Decision 3 - Option B): + the messages list is fresh per chunk, ensuring full parallel independence + while being ready for per-chunk retries in future extensions. + """ + messages: list[dict] = [ + {"role": "system", "content": _chunk_system(track)}, + { + "role": "user", + "content": ( + f"Migriere Chunk #{chunk.index} " + f"(Zeilen {chunk.start_line}\u2013{chunk.end_line - 1}):\n\n{chunk.content}" + ), + }, + ] + reply = await call_api_multiturn(client, model, messages) + messages.append({"role": "assistant", "content": reply}) + return reply + + +async def _consistency_pass_mt( + client: httpx.AsyncClient, + model: str, + merged: str, +) -> str: + """Consistency pass with its own fresh multi-turn conversation. + + Uses a dedicated message list (Decision 1 - Option B scope), independent + of the whole-file retry conversation, so it can be extended with its own + retry turns in future without cross-contaminating the migration history. + """ + messages: list[dict] = [ + {"role": "system", "content": _SYSTEM_MERGE_CHECK}, + {"role": "user", "content": f"Pr\u00fcfe und korrigiere:\n\n{merged}"}, + ] + reply = await call_api_multiturn(client, model, messages) + messages.append({"role": "assistant", "content": reply}) + return reply + + +# --------------------------------------------------------------------------- +# Budget helpers +# --------------------------------------------------------------------------- + +def _hard_budget(line_count: int) -> float: + return HARD_BUDGET_LARGE_FILE if line_count > LARGE_FILE_THRESHOLD else HARD_BUDGET_SMALL_FILE + + +# --------------------------------------------------------------------------- +# Core agent +# --------------------------------------------------------------------------- + +async def migrate_file( + *, + client: httpx.AsyncClient, + source_file: Path, # absolute path inside the worktree + worktree_root: Path, + build_dir: Path, + track: Track, + batch_id: str, +) -> FileResult: + """Migrate one file and return a FileResult.""" + file_str = str(source_file) + original_code = source_file.read_text(errors="ignore") + line_count = len(original_code.splitlines()) + model = select_model(line_count) + budget = _hard_budget(line_count) + is_large = line_count > CHUNK_SIZE_LINES + chunks = split_semantic(original_code, CHUNK_SIZE_LINES) if is_large else [] + chunk_info = ( + f"{len(chunks)} chunks, ~{CHUNK_SIZE_LINES} Zeilen/Chunk" if chunks + else f"vollst\u00e4ndig ({line_count} Zeilen)" + ) + + start = time.monotonic() + current_code = original_code + error_context = "" + last_gate = QualityGateResult() + iteration = 0 + test_prefix: str | None = None + + # Multi-turn conversation for whole-file migrations. + # Decision 4 (Option A): strictly per-file - this list is local and + # discarded when migrate_file returns. + whole_messages: list[dict] = [] + + while iteration < MAX_RETRIES: + elapsed = time.monotonic() - start + + # --- Hard budget check -------------------------------------------- + if elapsed >= budget: + logger.warning( + "[BUDGET] %s [%s] \u2013 %.0f s (budget %.0f s)", + source_file.name, track.value, elapsed, budget, + ) + break + + # --- Soft budget warning ------------------------------------------ + if elapsed >= SOFT_BUDGET_PER_FILE and iteration == 0: + logger.warning( + "[SOFT BUDGET] %s [%s] exceeded soft limit at %.0f s", + source_file.name, track.value, elapsed, + ) + + try: + # ---- 1. Migrate ------------------------------------------------ + if is_large and iteration == 0: + # Parallel chunk migration: each chunk gets its own fresh + # multi-turn messages list (Decision 3 - Option B). + migrated_parts = await asyncio.gather( + *[_migrate_chunk_mt(client, model, c, track) for c in chunks] + ) + merged = "\n".join(migrated_parts) + # Consistency pass uses its own independent multi-turn + # conversation (Decision 1 - Option B). + migrated_code = await _consistency_pass_mt(client, model, merged) + else: + # Whole-file path: conversation grows per retry so the model + # sees its own previous answer alongside the compiler error + # (Decision 1 - Option B, Decision 4 - Option A). + migrated_code = await _migrate_whole_mt( + client, model, current_code, track, whole_messages, error_context + ) + + # ---- 2. Write migrated file to worktree ----------------------- + source_file.write_text(migrated_code) + + # ---- 3. Fast per-file syntax check ---------------------------- + syntax_ok, syn_out, syn_err = await syntax_check_file( + source_file, build_dir + ) + syntax_warnings = parse_warnings(syn_out + "\n" + syn_err) + + if not syntax_ok: + logger.info( + "[SYNTAX FAIL] %s [%s] iter %d", + source_file.name, track.value, iteration, + ) + error_context = f"Compiler-Syntaxfehler:\n{syn_err}" + current_code = migrated_code + iteration += 1 + continue + + # ---- 4. Generate tests ---------------------------------------- + logger.info( + "[QA] Generating tests for %s [%s]...", + source_file.name, track.value, + ) + test_prefix = await generate_tests( + client, model, file_str, migrated_code, + worktree_root, track.value, + ) + + # ---- 5. Reconfigure + full build (to include new tests) ------- + await cmake_configure(worktree_root, build_dir) + last_gate = await evaluate_gates(build_dir, test_prefix) + + # Merge syntax-check warnings into gate result. + last_gate.build_warnings = syntax_warnings + last_gate.build_warnings + + if last_gate.all_gates_passed: + elapsed_final = time.monotonic() - start + logger.info( + "[SUCCESS] %s [%s] in %.0f s \u2013 %d warning(s)", + source_file.name, track.value, elapsed_final, + last_gate.total_warnings, + ) + # Commit migrated file and generated tests. + commit_migrated_file(source_file, worktree_root, track.value) + tests_dir = worktree_root / "tests_generated" + commit_generated_tests( + tests_dir, worktree_root, track.value, test_prefix + ) + return FileResult( + file_path=file_str, + track=track.value, + status=FileStatus.SUCCESS.value, + iterations=iteration + 1, + elapsed_seconds=round(elapsed_final, 1), + gate=last_gate, + migrated_path=file_str, + test_name=test_prefix or "", + ) + + # Gates failed - build error or test failure. + error_context = ( + f"Build-Fehler:\n{last_gate.build_errors}\n\n" + f"Test-Ausgabe:\n{last_gate.test_output}" + ) + logger.info( + "[GATE FAIL] %s [%s] iter %d \u2013 build_ok=%s, " + "existing_ok=%s, new_ok=%s", + source_file.name, track.value, iteration, + last_gate.build_ok, last_gate.existing_tests_ok, + last_gate.new_tests_ok, + ) + current_code = migrated_code + iteration += 1 + + except Exception as exc: # noqa: BLE001 + logger.error( + "[ERROR] %s [%s] iter %d: %s", + source_file.name, track.value, iteration, exc, + ) + error_context = f"API/System-Fehler: {exc}" + # Restore original so next iteration starts clean. + source_file.write_text(original_code) + iteration += 1 + + # ---- Quarantine -------------------------------------------------------- + elapsed_total = time.monotonic() - start + logger.warning( + "[QUARANTINE] %s [%s] after %d iter, %.0f s", + source_file.name, track.value, iteration, elapsed_total, + ) + source_file.write_text(original_code) # restore + + q_entry = quarantine_file( + file_path=file_str, + track=track.value, + build_errors=last_gate.build_errors, + test_output=last_gate.test_output, + last_code=current_code, + elapsed=elapsed_total, + budget=budget, + retry_count=iteration, + chunk_info=chunk_info, + ) + + return FileResult( + file_path=file_str, + track=track.value, + status=FileStatus.QUARANTINED.value, + iterations=iteration, + elapsed_seconds=round(elapsed_total, 1), + gate=last_gate, + quarantine_entry=q_entry, + ) diff --git a/app/migration_pipeline/models.py b/app/migration_pipeline/models.py new file mode 100644 index 000000000..93ba73f80 --- /dev/null +++ b/app/migration_pipeline/models.py @@ -0,0 +1,151 @@ +"""Data models for the migration pipeline.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Optional + + +# --------------------------------------------------------------------------- +# Enumerations +# --------------------------------------------------------------------------- + +class Track(str, Enum): + """Migration track.""" + A = "track_a_conservative" # Qt5→Qt6 compatibility only + B = "track_b_modern" # Qt6 + C++20 modernisation + + +class FileStatus(str, Enum): + PENDING = "pending" + IN_PROGRESS = "in_progress" + SUCCESS = "success" + QUARANTINED = "quarantined" + + +class FailureClass(str, Enum): + COMPILE_ERROR = "compile_error" + LINK_ERROR = "link_error" + TEST_FAILURE = "test_failure" + TEST_BUILD_FAILURE = "test_build_failure" + BUDGET_EXCEEDED = "budget_exceeded" + API_ERROR = "api_error" + CHUNK_MERGE_ERROR = "chunk_merge_error" + UNKNOWN = "unknown" + + +# --------------------------------------------------------------------------- +# Chunk +# --------------------------------------------------------------------------- + +@dataclass +class CodeChunk: + """A contiguous slice of a C++ source file.""" + index: int + start_line: int # 0-based, inclusive + end_line: int # 0-based, exclusive + content: str + boundary_type: str # "complete" | "semantic" | "fallback" + + @property + def line_count(self) -> int: + return self.end_line - self.start_line + + +# --------------------------------------------------------------------------- +# Quality gates +# --------------------------------------------------------------------------- + +@dataclass +class QualityGateResult: + """Result of all quality gates for one migration attempt.""" + build_ok: bool = False + build_errors: str = "" + build_warnings: list[str] = field(default_factory=list) + + existing_tests_ok: bool = False + new_tests_ok: bool = False + test_output: str = "" + test_warnings: list[str] = field(default_factory=list) + + @property + def all_gates_passed(self) -> bool: + """Hard gates: build green, all tests green. + Warnings are advisory and do NOT block (config.WARNINGS_ARE_BLOCKING=False).""" + from migration_pipeline.config import WARNINGS_ARE_BLOCKING + if WARNINGS_ARE_BLOCKING: + return ( + self.build_ok + and self.existing_tests_ok + and self.new_tests_ok + and not self.build_warnings + and not self.test_warnings + ) + return self.build_ok and self.existing_tests_ok and self.new_tests_ok + + @property + def total_warnings(self) -> int: + return len(self.build_warnings) + len(self.test_warnings) + + +# --------------------------------------------------------------------------- +# Quarantine +# --------------------------------------------------------------------------- + +@dataclass +class QuarantineEntry: + """Rich metadata for a file that could not be migrated automatically.""" + file_path: str + track: str + failure_class: str + last_error: str # truncated at 2000 chars + affected_apis: list[str] + chunk_info: str + manual_recommendation: str + timestamp: str # ISO-8601 UTC + retry_count: int + elapsed_seconds: float + + +# --------------------------------------------------------------------------- +# Per-file result +# --------------------------------------------------------------------------- + +@dataclass +class FileResult: + file_path: str + track: str + status: str # FileStatus value + iterations: int + elapsed_seconds: float + gate: QualityGateResult + quarantine_entry: Optional[QuarantineEntry] = None + migrated_path: str = "" + test_name: str = "" + + +# --------------------------------------------------------------------------- +# Batch result +# --------------------------------------------------------------------------- + +@dataclass +class BatchResult: + """Aggregated result for one batch + one track.""" + batch_id: str + track: str + branch: str + total_files: int + success_count: int = 0 + quarantine_count: int = 0 + total_warnings: int = 0 + mean_elapsed_seconds: float = 0.0 + results: list[FileResult] = field(default_factory=list) + + @property + def success_rate(self) -> float: + return self.success_count / self.total_files if self.total_files else 0.0 + + @property + def quarantine_rate(self) -> float: + return self.quarantine_count / self.total_files if self.total_files else 0.0 diff --git a/app/migration_pipeline/orchestrator.py b/app/migration_pipeline/orchestrator.py new file mode 100644 index 000000000..7e501c68b --- /dev/null +++ b/app/migration_pipeline/orchestrator.py @@ -0,0 +1,204 @@ +"""Batch orchestrator – coordinates the full migration of one batch. + +Flow: + 1. Read the prioritised file list (one path per line, in order). + 2. Create git worktrees for Track A and Track B in parallel. + 3. Run cmake --configure in each worktree. + 4. Process files in priority order, with at most MAX_CONCURRENT_TASKS + workers per track. Both tracks run concurrently. + 5. Commit a batch-summary to each branch. + 6. Produce a comparison report (Track A vs Track B). +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from pathlib import Path + +import httpx + +from migration_pipeline.build_runner import cmake_configure +from migration_pipeline.config import ( + MAX_CONCURRENT_TASKS, + REPORTS_DIR, + TRACK_A_ID, + TRACK_B_ID, + WORKSPACE_ROOT, +) +from migration_pipeline.git_manager import ( + commit_batch_summary, + setup_worktree, +) +from migration_pipeline.migrator import migrate_file +from migration_pipeline.models import BatchResult, FileResult, FileStatus, Track +from migration_pipeline.reporter import write_comparison_report + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# File-list loading +# --------------------------------------------------------------------------- + +def load_file_list(list_path: Path, source_root: Path) -> list[str]: + """Read the external priority file list. + + Accepts: + - Plain text: one file path per line (blank lines and #-comments ignored). + - The paths may be absolute or relative to source_root. + """ + base = source_root.resolve() + files: list[str] = [] + for raw in list_path.read_text().splitlines(): + stripped = raw.strip() + if not stripped or stripped.startswith("#"): + continue + p = Path(stripped) + if not p.is_absolute(): + p = base / p + files.append(str(p.resolve())) + return files + + +# --------------------------------------------------------------------------- +# Single-track worker pool +# --------------------------------------------------------------------------- + +async def _run_track( + client: httpx.AsyncClient, + file_list: list[str], + source_root: Path, + track: Track, + worktree_root: Path, + build_dir: Path, + batch_id: str, +) -> BatchResult: + semaphore = asyncio.Semaphore(MAX_CONCURRENT_TASKS) + source_root = source_root.resolve() + + async def _worker(fp: str) -> FileResult: + async with semaphore: + abs_file = Path(fp).resolve() + try: + rel_file = abs_file.relative_to(source_root) + except ValueError as exc: + raise ValueError( + f"File '{abs_file}' is not under source-root '{source_root}'" + ) from exc + source_file = worktree_root / rel_file + return await migrate_file( + client=client, + source_file=source_file, + worktree_root=worktree_root, + build_dir=build_dir, + track=track, + batch_id=batch_id, + ) + + logger.info("[TRACK %s] Starting %d files", track.value, len(file_list)) + results: list[FileResult] = await asyncio.gather( + *[_worker(fp) for fp in file_list] + ) + + successes = [r for r in results if r.status == FileStatus.SUCCESS.value] + quarantine = [r for r in results if r.status == FileStatus.QUARANTINED.value] + elapsed = [r.elapsed_seconds for r in results] + warnings = sum(r.gate.total_warnings for r in results) + + return BatchResult( + batch_id=batch_id, + track=track.value, + branch=f"migration/{batch_id}/{track.value}", + total_files=len(results), + success_count=len(successes), + quarantine_count=len(quarantine), + total_warnings=warnings, + mean_elapsed_seconds=sum(elapsed) / len(elapsed) if elapsed else 0.0, + results=results, + ) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + +async def run_batch( + batch_id: str, + file_list_path: Path, + repo_root: Path, + source_root: Path, +) -> tuple[BatchResult, BatchResult]: + """Run the full batch on both tracks in parallel. + + Returns ``(result_track_a, result_track_b)``. + """ + file_list = load_file_list(file_list_path, source_root) + logger.info( + "[BATCH %s] Loaded %d files from %s", + batch_id, len(file_list), file_list_path, + ) + + workspace = WORKSPACE_ROOT.resolve() + REPORTS_DIR.mkdir(parents=True, exist_ok=True) + + # ---- Set up worktrees for both tracks ---------------------------------- + worktree_a = setup_worktree(batch_id, TRACK_A_ID, repo_root, workspace) + worktree_b = setup_worktree(batch_id, TRACK_B_ID, repo_root, workspace) + + build_dir_a = worktree_a / "build" + build_dir_b = worktree_b / "build" + + # ---- Configure cmake in both worktrees (in parallel) ------------------ + logger.info("[BATCH %s] Configuring cmake for both tracks...", batch_id) + await asyncio.gather( + cmake_configure(worktree_a, build_dir_a), + cmake_configure(worktree_b, build_dir_b), + ) + + # ---- Run both tracks concurrently ------------------------------------- + async with httpx.AsyncClient() as client: + result_a, result_b = await asyncio.gather( + _run_track( + client, file_list, source_root, Track.A, worktree_a, build_dir_a, batch_id + ), + _run_track( + client, file_list, source_root, Track.B, worktree_b, build_dir_b, batch_id + ), + ) + + # ---- Final batch commit on each branch -------------------------------- + commit_batch_summary(worktree_a, TRACK_A_ID, batch_id) + commit_batch_summary(worktree_b, TRACK_B_ID, batch_id) + + # ---- Write comparison report ------------------------------------------ + report_path = REPORTS_DIR / f"{batch_id}_comparison.md" + write_comparison_report(result_a, result_b, report_path) + logger.info("[BATCH %s] Report written to %s", batch_id, report_path) + + _print_summary(result_a, result_b) + return result_a, result_b + + +# --------------------------------------------------------------------------- +# Console summary +# --------------------------------------------------------------------------- + +def _print_summary(a: BatchResult, b: BatchResult) -> None: + print("\n" + "=" * 64) + print(f" Batch: {a.batch_id}") + print("=" * 64) + _print_track_row("Track A (konservativ)", a) + _print_track_row("Track B (modern) ", b) + print("=" * 64 + "\n") + + +def _print_track_row(label: str, r: BatchResult) -> None: + print( + f" {label}: " + f"{r.success_count}/{r.total_files} OK | " + f"{r.quarantine_count} quarantined | " + f"{r.total_warnings} warnings | " + f"avg {r.mean_elapsed_seconds:.0f}s" + ) diff --git a/app/migration_pipeline/quality_gate.py b/app/migration_pipeline/quality_gate.py new file mode 100644 index 000000000..30b9596fa --- /dev/null +++ b/app/migration_pipeline/quality_gate.py @@ -0,0 +1,56 @@ +"""Quality gate evaluation. + +Gates (in order): + 1. Build green – cmake --build succeeds with zero errors + 2. Existing tests green – full CTest suite passes + 3. New tests green – only the newly generated tests pass + +Warnings are always collected and stored for reporting, but they are +advisory only (do NOT block) unless config.WARNINGS_ARE_BLOCKING=True. +""" + +from __future__ import annotations + +from pathlib import Path + +from migration_pipeline.build_runner import cmake_build, parse_warnings, run_ctest +from migration_pipeline.models import QualityGateResult + + +async def evaluate_gates( + build_dir: Path, + test_prefix: str | None = None, +) -> QualityGateResult: + """Run all quality gates and return an aggregated result. + + *test_prefix* is the CTest regex for newly generated tests. + If None, the new-tests gate is skipped (considered passing). + """ + gate = QualityGateResult() + + # --- Gate 1: Build ------------------------------------------------------ + build_ok, build_out, build_err = await cmake_build(build_dir) + gate.build_ok = build_ok + gate.build_errors = build_err + gate.build_warnings = parse_warnings(build_out + "\n" + build_err) + + if not build_ok: + # Short-circuit: remaining gates cannot run without a successful build. + return gate + + # --- Gate 2: Existing tests --------------------------------------------- + existing_ok, ctest_out, ctest_err = await run_ctest(build_dir) + gate.existing_tests_ok = existing_ok + gate.test_output = ctest_out + "\n" + ctest_err + gate.test_warnings = parse_warnings(ctest_err) + + # --- Gate 3: New tests only --------------------------------------------- + if test_prefix: + new_ok, new_out, new_err = await run_ctest(build_dir, test_prefix) + gate.new_tests_ok = new_ok + gate.test_output += f"\n--- New tests ({test_prefix}) ---\n{new_out}{new_err}" + gate.test_warnings += parse_warnings(new_err) + else: + gate.new_tests_ok = True # no new test → gate vacuously true + + return gate diff --git a/app/migration_pipeline/quarantine.py b/app/migration_pipeline/quarantine.py new file mode 100644 index 000000000..9a62d17df --- /dev/null +++ b/app/migration_pipeline/quarantine.py @@ -0,0 +1,190 @@ +"""Quarantine management. + +When a file cannot be migrated within the retry budget, it is: + 1. Written to ``quarantine/__.json`` (machine-readable) + 2. Written to ``quarantine/__.md`` (manual-review ticket) + +The Markdown ticket is the automatically generated "manual ticket" requested +in the pipeline requirements. +""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timezone +from pathlib import Path + +from migration_pipeline.config import QUARANTINE_DIR +from migration_pipeline.models import FailureClass, QuarantineEntry + + +# --------------------------------------------------------------------------- +# Qt5 deprecated-API detector +# --------------------------------------------------------------------------- + +# A non-exhaustive list of Qt5 APIs that are removed / changed in Qt6. +_QT5_DEPRECATED_RE = re.compile( + r"\b(" + r"QRegExp|QTextCodec|QMatrix|QDesktopWidget|" + r"QSignalMapper|QSortFilterProxyModel::lessThan|" + r"QAbstractItemModel::reset|" + r"QApplication::desktop|" + r"QFontMetrics::width\s*\(|" + r"QUrl::addQueryItem|" + r"QSettings::setDefaultFormat|" + r"QVariant::type\b|" + r"QMetaType::type\b|" + r"QTest::qSleep" + r")\b" +) + + +def _extract_affected_apis(code: str) -> list[str]: + return sorted(set(_QT5_DEPRECATED_RE.findall(code))) + + +# --------------------------------------------------------------------------- +# Failure classification +# --------------------------------------------------------------------------- + +def _classify( + build_errors: str, + test_output: str, + elapsed: float, + budget: float, +) -> FailureClass: + if elapsed >= budget: + return FailureClass.BUDGET_EXCEEDED + lower_err = build_errors.lower() + if "undefined reference" in lower_err or "linker command failed" in lower_err: + return FailureClass.LINK_ERROR + if "error:" in build_errors: + return FailureClass.COMPILE_ERROR + if "failed" in test_output.lower(): + return FailureClass.TEST_FAILURE + return FailureClass.UNKNOWN + + +def _recommend(fc: FailureClass, affected_apis: list[str]) -> str: + api_str = ", ".join(affected_apis) if affected_apis else "unbekannte APIs" + if fc == FailureClass.BUDGET_EXCEEDED: + return ( + "Datei hat das Zeitbudget überschritten. " + "Empfehlung: Datei manuell in kleinere, unabhängige Units zerlegen und erneut starten." + ) + if fc == FailureClass.COMPILE_ERROR: + return ( + f"Compilerfehler. Prüfe Qt6-API-Änderungen für: {api_str}. " + "Porting Guide: https://doc.qt.io/qt-6/portingguide.html" + ) + if fc == FailureClass.LINK_ERROR: + return ( + "Linkfehler. Prüfe Qt6-Modulabhängigkeiten in CMakeLists.txt " + "(z.B. find_package(Qt6 ...) und target_link_libraries)." + ) + if fc == FailureClass.TEST_FAILURE: + return ( + "Semantische Abweichung in Tests. Manuelle Überprüfung der Fachlogik erforderlich. " + "Vergleiche Qt5- und Qt6-Verhalten für betroffene APIs." + ) + if fc == FailureClass.CHUNK_MERGE_ERROR: + return ( + "Chunk-Merge ist inkonsistent. " + "Empfehlung: Datei als Ganzes manuell migrieren oder Chunk-Grenzen anpassen." + ) + return "Unbekannter Fehler. Bitte Logs manuell inspizieren." + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def quarantine_file( + *, + file_path: str, + track: str, + build_errors: str, + test_output: str, + last_code: str, + elapsed: float, + budget: float, + retry_count: int, + chunk_info: str, +) -> QuarantineEntry: + """Create a QuarantineEntry and write JSON + Markdown ticket to disk.""" + now = datetime.now(tz=timezone.utc) + ts = now.strftime("%Y%m%dT%H%M%SZ") + iso_ts = now.isoformat() + + fc = _classify(build_errors, test_output, elapsed, budget) + apis = _extract_affected_apis(last_code) + recommendation = _recommend(fc, apis) + + entry = QuarantineEntry( + file_path=file_path, + track=track, + failure_class=fc.value, + last_error=(build_errors or test_output)[:2000], + affected_apis=apis, + chunk_info=chunk_info, + manual_recommendation=recommendation, + timestamp=iso_ts, + retry_count=retry_count, + elapsed_seconds=round(elapsed, 1), + ) + + _write_quarantine_files(entry, ts) + return entry + + +# --------------------------------------------------------------------------- +# File writers +# --------------------------------------------------------------------------- + +def _write_quarantine_files(entry: QuarantineEntry, ts: str) -> None: + QUARANTINE_DIR.mkdir(parents=True, exist_ok=True) + stem = Path(entry.file_path).stem + base_name = f"{stem}_{entry.track}_{ts}" + + # --- JSON (machine-readable) ------------------------------------------- + json_path = QUARANTINE_DIR / f"{base_name}.json" + json_path.write_text(json.dumps(entry.__dict__, indent=2, ensure_ascii=False)) + + # --- Markdown ticket (manual review) ------------------------------------ + md_path = QUARANTINE_DIR / f"{base_name}.md" + md_path.write_text(_render_ticket(entry)) + + +def _render_ticket(e: QuarantineEntry) -> str: + apis_md = "\n".join(f"- `{a}`" for a in e.affected_apis) or "_keine erkannt_" + error_block = e.last_error.strip() or "_kein Fehlertext_" + return f"""# Quarantäne-Ticket: {Path(e.file_path).name} + +| Feld | Wert | +|---|---| +| **Datei** | `{e.file_path}` | +| **Track** | `{e.track}` | +| **Fehlerklasse** | `{e.failure_class}` | +| **Versuche** | {e.retry_count} | +| **Laufzeit** | {e.elapsed_seconds:.1f} s | +| **Zeitstempel** | {e.timestamp} | +| **Chunk-Info** | {e.chunk_info} | + +## Letzte Fehlermeldung + +``` +{error_block} +``` + +## Betroffene Qt5-APIs + +{apis_md} + +## Empfehlung für manuelle Migration + +{e.manual_recommendation} + +--- +_Automatisch generiert von der Qt5→Qt6 Migrationspipeline._ +""" diff --git a/app/migration_pipeline/reporter.py b/app/migration_pipeline/reporter.py new file mode 100644 index 000000000..0dce87a9a --- /dev/null +++ b/app/migration_pipeline/reporter.py @@ -0,0 +1,221 @@ +"""Batch comparison reporter. + +Generates a Markdown report that places Track A and Track B side-by-side +so the burn-phase decision can be made on objective metrics. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from migration_pipeline.models import BatchResult, FileResult, FileStatus + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def write_comparison_report( + result_a: BatchResult, + result_b: BatchResult, + output_path: Path, +) -> None: + """Write a Markdown comparison of Track A vs Track B to *output_path*.""" + output_path.parent.mkdir(parents=True, exist_ok=True) + md = _render_markdown(result_a, result_b) + output_path.write_text(md, encoding="utf-8") + + # Machine-readable companion file. + json_path = output_path.with_suffix(".json") + json_path.write_text( + json.dumps(_to_dict(result_a, result_b), indent=2, ensure_ascii=False), + encoding="utf-8", + ) + + +# --------------------------------------------------------------------------- +# Markdown rendering +# --------------------------------------------------------------------------- + +def _render_markdown(a: BatchResult, b: BatchResult) -> str: + ts = _utc_now() + lines: list[str] = [ + f"# Migrationsbericht – Batch `{a.batch_id}`", + f"_Erstellt: {ts}_", + "", + "## Übersicht", + "", + "| Metrik | Track A – konservativ | Track B – modern |", + "|---|---|---|", + f"| Branch | `{a.branch}` | `{b.branch}` |", + f"| Dateien gesamt | {a.total_files} | {b.total_files} |", + f"| Erfolge | {a.success_count} ({a.success_rate:.0%}) | {b.success_count} ({b.success_rate:.0%}) |", + f"| Quarantäne | {a.quarantine_count} ({a.quarantine_rate:.0%}) | {b.quarantine_count} ({b.quarantine_rate:.0%}) |", + f"| Warnungen (gesamt, advisory) | {a.total_warnings} | {b.total_warnings} |", + f"| Mittlere Laufzeit | {a.mean_elapsed_seconds:.0f} s | {b.mean_elapsed_seconds:.0f} s |", + "", + "## Datei-Detailtabelle", + "", + "| Datei | Track A Status | Track A Zeit | Track A Warn | Track B Status | Track B Zeit | Track B Warn |", + "|---|---|---|---|---|---|---|", + ] + + # Build lookup for track B by file path. + b_by_file = {r.file_path: r for r in b.results} + + for ra in a.results: + rb = b_by_file.get(ra.file_path) + name = Path(ra.file_path).name + lines.append( + f"| `{name}` " + f"| {_status_badge(ra.status)} " + f"| {ra.elapsed_seconds:.0f} s " + f"| {ra.gate.total_warnings} " + f"| {_status_badge(rb.status) if rb else '—'} " + f"| {f'{rb.elapsed_seconds:.0f} s' if rb else '—'} " + f"| {rb.gate.total_warnings if rb else '—'} |" + ) + + lines += [ + "", + "## Quarantäne-Dateien", + "", + ] + + quarantine_a = [r for r in a.results if r.status == FileStatus.QUARANTINED.value] + quarantine_b = [r for r in b.results if r.status == FileStatus.QUARANTINED.value] + + if quarantine_a or quarantine_b: + lines += [ + "| Datei | Track | Fehlerklasse | Versuche | Empfehlung |", + "|---|---|---|---|---|", + ] + for r in quarantine_a + quarantine_b: + q = r.quarantine_entry + if q: + name = Path(r.file_path).name + rec = q.manual_recommendation.replace("|", "\\|")[:80] + lines.append( + f"| `{name}` | {r.track} | `{q.failure_class}` " + f"| {q.retry_count} | {rec}… |" + ) + else: + lines.append("_Keine Dateien in Quarantäne._") + + lines += [ + "", + "## Warnungen (Advisory)", + "", + "> Warnungen blockieren keine Quality Gates. Sie sind hier zur " + "Entscheidungshilfe für die Burn-Phase aufgeführt.", + "", + ] + + warn_rows = [ + r for r in (a.results + b.results) if r.gate.total_warnings > 0 + ] + if warn_rows: + lines += [ + "| Datei | Track | Anzahl Warnungen |", + "|---|---|---|", + ] + for r in sorted(warn_rows, key=lambda x: -x.gate.total_warnings): + lines.append( + f"| `{Path(r.file_path).name}` | {r.track} " + f"| {r.gate.total_warnings} |" + ) + else: + lines.append("_Keine Warnungen aufgetreten._") + + lines += [ + "", + "## Burn-Phase Empfehlung", + "", + _burn_recommendation(a, b), + "", + "---", + "_Automatisch generiert von der Qt5→Qt6 Migrationspipeline._", + ] + + return "\n".join(lines) + + +def _status_badge(status: str) -> str: + icons = { + FileStatus.SUCCESS.value: "✅ Erfolg", + FileStatus.QUARANTINED.value: "🔴 Quarantäne", + FileStatus.IN_PROGRESS.value: "⏳ Läuft", + FileStatus.PENDING.value: "⏸ Ausstehend", + } + return icons.get(status, status) + + +def _burn_recommendation(a: BatchResult, b: BatchResult) -> str: + if a.success_rate > b.success_rate + 0.05: + return ( + "**Track A (konservativ)** hat eine höhere Erfolgsquote. " + "Empfehlung für die Burn-Phase: Track A als Basis, Track B für " + "ausgewählte, gut getestete Module." + ) + if b.success_rate > a.success_rate + 0.05: + return ( + "**Track B (modern)** hat eine höhere Erfolgsquote. " + "Empfehlung: Track B als Basis übernehmen." + ) + if a.total_warnings < b.total_warnings: + return ( + "Beide Tracks haben ähnliche Erfolgsquoten. " + "Track A hat weniger Warnungen – bevorzuge Track A für risikoarme Module, " + "Track B für Module mit guter Testabdeckung." + ) + return ( + "Beide Tracks sind gleichwertig. " + "Entscheidung liegt beim Team – empfohlene Kriterien: " + "Wartbarkeit, Warnungsfreiheit, vorhandene Testabdeckung." + ) + + +# --------------------------------------------------------------------------- +# JSON serialisation helper +# --------------------------------------------------------------------------- + +def _to_dict(a: BatchResult, b: BatchResult) -> dict: + def _result(r: BatchResult) -> dict: + return { + "batch_id": r.batch_id, + "track": r.track, + "branch": r.branch, + "total_files": r.total_files, + "success_count": r.success_count, + "quarantine_count": r.quarantine_count, + "total_warnings": r.total_warnings, + "mean_elapsed_seconds": round(r.mean_elapsed_seconds, 1), + "success_rate": round(r.success_rate, 4), + "quarantine_rate": round(r.quarantine_rate, 4), + "files": [_file_result(f) for f in r.results], + } + + def _file_result(f: FileResult) -> dict: + d: dict = { + "file_path": f.file_path, + "status": f.status, + "iterations": f.iterations, + "elapsed_seconds": f.elapsed_seconds, + "warnings": f.gate.total_warnings, + "test_name": f.test_name, + } + if f.quarantine_entry: + d["quarantine"] = f.quarantine_entry.__dict__ + return d + + return {"track_a": _result(a), "track_b": _result(b)} + + +# --------------------------------------------------------------------------- +# Utility +# --------------------------------------------------------------------------- + +def _utc_now() -> str: + from datetime import datetime, timezone + return datetime.now(tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") diff --git a/app/migration_pipeline/test_generator.py b/app/migration_pipeline/test_generator.py new file mode 100644 index 000000000..00f5aab64 --- /dev/null +++ b/app/migration_pipeline/test_generator.py @@ -0,0 +1,156 @@ +"""Test-file generation for migrated C++ code. + +Strategy (B + C combined, no pre-existing test infrastructure): + - QTest for Qt-heavy translation units (QObject / signals / slots) + - gtest for pure-logic / non-UI code + - Smoke CTest for every file (build + link + default-construction check) + +Generated files are written to ``{worktree_root}/tests_generated/`` and +registered in that directory's CMakeLists.txt (idempotent via a guard +comment). +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path + +import httpx + +from migration_pipeline.api_client import call_api + + +# --------------------------------------------------------------------------- +# System prompts +# --------------------------------------------------------------------------- + +_QTEST_SYSTEM = ( + "Du bist ein QA-Ingenieur für Qt6/C++20. " + "Schreibe einen eigenständigen QTest Unit-Test für den folgenden migrierten Qt6-Code. " + "Verwende QCOMPARE, QVERIFY und QTEST_MAIN. " + "Gib NUR den C++ Quellcode zurück, ohne Markdown oder Erklärungen." +) + +_GTEST_SYSTEM = ( + "Du bist ein QA-Ingenieur für C++20. " + "Schreibe einen eigenständigen Google Test (gtest) Unit-Test für den folgenden C++ Code. " + "Gib NUR den C++ Quellcode zurück, ohne Markdown oder Erklärungen." +) + +_SMOKE_SYSTEM = ( + "Du bist ein QA-Ingenieur. " + "Schreibe einen minimalen Smoke-Test mit Google Test, der sicherstellt, dass die Datei " + "kompiliert, linkt und die Default-Konstruktion der Hauptklasse nicht crasht. " + "Gib NUR den C++ Quellcode zurück, ohne Markdown oder Erklärungen." +) + + +# --------------------------------------------------------------------------- +# Qt-content heuristic +# --------------------------------------------------------------------------- + +_QT_MARKERS = ( + "QObject", + "QWidget", + "QApplication", + "QMainWindow", + "emit ", + "signals:", + "slots:", + "Q_OBJECT", + "Q_EMIT", +) + + +def _is_qt_heavy(code: str) -> bool: + return any(marker in code for marker in _QT_MARKERS) + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +async def generate_tests( + client: httpx.AsyncClient, + model: str, + source_file_path: str, + migrated_code: str, + worktree_root: Path, + track_id: str, +) -> str: + """Generate unit + smoke tests for *migrated_code*. + + Returns the CTest test-name prefix (e.g. ``test_track_a_conservative_MyClass``). + Both the unit test and smoke test are registered under that prefix. + """ + base_name = Path(source_file_path).stem + test_prefix = f"test_{track_id}_{base_name}" + tests_dir = worktree_root / "tests_generated" + tests_dir.mkdir(parents=True, exist_ok=True) + + use_qtest = _is_qt_heavy(migrated_code) + unit_system = _QTEST_SYSTEM if use_qtest else _GTEST_SYSTEM + framework = "Qt6::Core Qt6::Test" if use_qtest else "GTest::gtest_main" + + # Generate unit test and smoke test in parallel. + unit_code, smoke_code = await asyncio.gather( + call_api( + client, model, unit_system, + f"Erstelle einen Unit-Test für folgenden migrierten Qt6-Code:\n\n{migrated_code}", + ), + call_api( + client, model, _SMOKE_SYSTEM, + f"Erstelle einen Smoke-Test für:\n\n{migrated_code}", + ), + ) + + unit_file = tests_dir / f"{test_prefix}_unit.cpp" + smoke_file = tests_dir / f"{test_prefix}_smoke.cpp" + unit_file.write_text(unit_code) + smoke_file.write_text(smoke_code) + + _register_tests_in_cmake( + tests_dir, + test_prefix, + unit_file.name, + smoke_file.name, + framework, + ) + + return test_prefix + + +# --------------------------------------------------------------------------- +# CMakeLists.txt registration (idempotent) +# --------------------------------------------------------------------------- + +_GUARD_PREFIX = "# MIGRATION_PIPELINE:" + + +def _register_tests_in_cmake( + tests_dir: Path, + test_prefix: str, + unit_filename: str, + smoke_filename: str, + framework_libs: str, +) -> None: + cmake_path = tests_dir / "CMakeLists.txt" + guard = f"{_GUARD_PREFIX}{test_prefix}" + + # Read existing content to avoid duplicate registrations. + existing = cmake_path.read_text() if cmake_path.exists() else "" + if guard in existing: + return # already registered + + entry = ( + f"\n{guard}\n" + f"add_executable({test_prefix}_unit {unit_filename})\n" + f"target_link_libraries({test_prefix}_unit PRIVATE {framework_libs})\n" + f"add_test(NAME {test_prefix}_unit COMMAND {test_prefix}_unit)\n" + f"\nadd_executable({test_prefix}_smoke {smoke_filename})\n" + f"target_link_libraries({test_prefix}_smoke PRIVATE GTest::gtest_main)\n" + f"add_test(NAME {test_prefix}_smoke COMMAND {test_prefix}_smoke)\n" + ) + + with cmake_path.open("a") as fh: + fh.write(entry) diff --git a/app/requirements.txt b/app/requirements.txt new file mode 100644 index 000000000..ae6efddd0 --- /dev/null +++ b/app/requirements.txt @@ -0,0 +1,2 @@ +httpx>=0.27.0 +python-dotenv>=1.0.0 diff --git a/app/run_docker.sh b/app/run_docker.sh new file mode 100755 index 000000000..29521bca7 --- /dev/null +++ b/app/run_docker.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +IMAGE_NAME="${IMAGE_NAME:-migration-pipeline:ubuntu22.04}" + +CONTAINER_USER="${CONTAINER_USER:-${USER:-runner}}" +CONTAINER_GROUP="${CONTAINER_GROUP:-$CONTAINER_USER}" +ENABLE_CONTAINER_SUDO="${ENABLE_CONTAINER_SUDO:-1}" +QT_ROOT_DEFAULT="${QT_ROOT_DEFAULT:-/opt/Qt/6.8.3/gcc_64}" + +GIT_REPO_DIR="" +WORKDIR_IN_CONTAINER="" +RUN_QT_SMOKE_TEST=0 +QT_ROOT_OVERRIDE="" +DOCKER_ARGS=() +CMD_ARGS=() + +usage() { + cat <<'EOF' +Usage: + ./run_docker.sh [options] [repo_dir] [-- command args...] + +Options: + --no-sudo Disable passwordless sudo for the container user + --workdir PATH Set working directory inside container + --qt-smoke-test Run a Qt/CMake smoke test inside the container + --qt-path PATH Override Qt root path inside container + -h, --help Show this help + +Examples: + ./run_docker.sh + ./run_docker.sh /path/to/repo + ./run_docker.sh /path/to/repo -- bash + ./run_docker.sh --qt-smoke-test /path/to/repo + ./run_docker.sh --qt-path /opt/Qt/6.8.3/gcc_64 /path/to/repo -- bash +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --no-sudo) + ENABLE_CONTAINER_SUDO=0 + shift + ;; + --workdir) + WORKDIR_IN_CONTAINER="$2" + shift 2 + ;; + --qt-smoke-test) + RUN_QT_SMOKE_TEST=1 + shift + ;; + --qt-path) + QT_ROOT_OVERRIDE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + CMD_ARGS=("$@") + break + ;; + -*) + echo "Unknown option: $1" >&2 + usage >&2 + exit 1 + ;; + *) + if [[ -z "$GIT_REPO_DIR" ]]; then + GIT_REPO_DIR="$1" + else + CMD_ARGS+=("$1") + fi + shift + ;; + esac +done + +HOST_UID="$(id -u)" +HOST_GID="$(id -g)" + +add_optional_mount() { + local host_path="$1" + local container_path="$2" + + if [[ -e "$host_path" ]]; then + DOCKER_ARGS+=( -v "$host_path:$container_path" ) + fi +} + +add_optional_mount "$HOME/.gitconfig" "/home/$CONTAINER_USER/.gitconfig" +add_optional_mount "$HOME/.ssh" "/home/$CONTAINER_USER/.ssh" +add_optional_mount "$HOME/.bashrc" "/home/$CONTAINER_USER/.bashrc" +add_optional_mount "$HOME/.bash_aliases" "/home/$CONTAINER_USER/.bash_aliases" + +if [[ -n "$GIT_REPO_DIR" ]]; then + if [[ ! -d "$GIT_REPO_DIR" ]]; then + echo "Repository directory does not exist: $GIT_REPO_DIR" >&2 + exit 1 + fi + + GIT_REPO_DIR="$(cd "$GIT_REPO_DIR" && pwd)" + DOCKER_ARGS+=( -v "$GIT_REPO_DIR:/repo" ) + + if [[ -d "$GIT_REPO_DIR/src" ]]; then + DOCKER_ARGS+=( -v "$GIT_REPO_DIR/src:/src" ) + fi + + if [[ -z "$WORKDIR_IN_CONTAINER" ]]; then + WORKDIR_IN_CONTAINER="/repo" + fi +fi + +if [[ -z "$WORKDIR_IN_CONTAINER" ]]; then + WORKDIR_IN_CONTAINER="/app" +fi + +if [[ -z "$QT_ROOT_OVERRIDE" ]]; then + QT_ROOT_OVERRIDE="$QT_ROOT_DEFAULT" +fi + +if [[ ! -f "$SCRIPT_DIR/.env" ]]; then + echo "Missing env file: $SCRIPT_DIR/.env" >&2 + exit 1 +fi + +docker build -t "$IMAGE_NAME" "$SCRIPT_DIR" + +if [[ $RUN_QT_SMOKE_TEST -eq 1 ]]; then + CMD_ARGS=( + bash -lc " + set -euo pipefail + echo '== User ==' + whoami + echo '== Qt path ==' + echo \"$QT_ROOT_OVERRIDE\" + test -d \"$QT_ROOT_OVERRIDE\" + echo '== CMake version ==' + cmake --version + echo '== Qt config ==' + find \"$QT_ROOT_OVERRIDE\" -maxdepth 3 \\( -name 'Qt6Config.cmake' -o -name 'Qt6CoreConfig.cmake' \\) | sort + TMPDIR=\$(mktemp -d) + cat > \"\$TMPDIR/CMakeLists.txt\" <<'EOF' +cmake_minimum_required(VERSION 3.21) +project(QtSmokeTest LANGUAGES CXX) +find_package(Qt6 REQUIRED COMPONENTS Core) +add_executable(qt_smoke main.cpp) +target_link_libraries(qt_smoke PRIVATE Qt6::Core) +EOF + cat > \"\$TMPDIR/main.cpp\" <<'EOF' +#include +#include +#include + +int main(int argc, char **argv) { + QCoreApplication app(argc, argv); + std::cout << QString(\"Qt smoke test OK\").toStdString() << std::endl; + return 0; +} +EOF + cmake -S \"\$TMPDIR\" -B \"\$TMPDIR/build\" -G Ninja -DCMAKE_PREFIX_PATH=\"$QT_ROOT_OVERRIDE\" + cmake --build \"\$TMPDIR/build\" + \"\$TMPDIR/build/qt_smoke\" + " + ) +fi + +if [[ ${#CMD_ARGS[@]} -eq 0 ]]; then + CMD_ARGS=(bash) +fi + +if [ -t 1 ]; then + DOCKER_TTY_FLAG="-it" +else + DOCKER_TTY_FLAG="-i" +fi + +docker run --rm $DOCKER_TTY_FLAG \ + --user root \ + --env-file "$SCRIPT_DIR/.env" \ + -v "$SCRIPT_DIR:/app" \ + "${DOCKER_ARGS[@]}" \ + -w "$WORKDIR_IN_CONTAINER" \ + -e LOCAL_UID="$HOST_UID" \ + -e LOCAL_GID="$HOST_GID" \ + -e LOCAL_USER="$CONTAINER_USER" \ + -e LOCAL_GROUP="$CONTAINER_GROUP" \ + -e ENABLE_CONTAINER_SUDO="$ENABLE_CONTAINER_SUDO" \ + -e WORKDIR_IN_CONTAINER="$WORKDIR_IN_CONTAINER" \ + -e QT_ROOT="$QT_ROOT_OVERRIDE" \ + -e Qt6_ROOT="$QT_ROOT_OVERRIDE" \ + -e CMAKE_PREFIX_PATH="$QT_ROOT_OVERRIDE" \ + "$IMAGE_NAME" \ + bash -s -- "${CMD_ARGS[@]}" <<'DOCKER_SCRIPT' +set -euo pipefail + +ensure_group() { + # If a group with the requested GID exists, use that group name. + if getent group "${LOCAL_GID}" >/dev/null 2>&1; then + getent group "${LOCAL_GID}" | cut -d: -f1 + return + fi + + # If the desired group name exists, try to change its GID if unused. + if getent group "${LOCAL_GROUP}" >/dev/null 2>&1; then + existing_gid=$(getent group "${LOCAL_GROUP}" | cut -d: -f3) + if [[ "${existing_gid}" != "${LOCAL_GID}" ]]; then + if ! getent group "${LOCAL_GID}" >/dev/null 2>&1; then + groupmod -g "${LOCAL_GID}" "${LOCAL_GROUP}" || true + else + echo "Warning: desired GID ${LOCAL_GID} already in use by another group; keeping ${LOCAL_GROUP} gid ${existing_gid}" >&2 + fi + fi + echo "${LOCAL_GROUP}" + return + fi + + # Otherwise create the group with requested GID, unless GID is taken. + if getent group "${LOCAL_GID}" >/dev/null 2>&1; then + # Some other group holds the target GID; return that group's name. + getent group "${LOCAL_GID}" | cut -d: -f1 + else + groupadd -g "${LOCAL_GID}" "${LOCAL_GROUP}" + echo "${LOCAL_GROUP}" + fi +} + +ensure_user() { + # If a user with the desired UID already exists, use that user instead + if getent passwd "${LOCAL_UID}" >/dev/null 2>&1; then + existing_user=$(getent passwd "${LOCAL_UID}" | cut -d: -f1) + echo "Host UID ${LOCAL_UID} already exists in container as user '${existing_user}'; will use that account." + LOCAL_USER="${existing_user}" + return + fi + + # If the desired username already exists, try to change its UID (only if the target UID is free) + if id -u "${LOCAL_USER}" >/dev/null 2>&1; then + current_uid=$(id -u "${LOCAL_USER}") + if [[ "${current_uid}" != "${LOCAL_UID}" ]]; then + if ! getent passwd "${LOCAL_UID}" >/dev/null 2>&1; then + usermod -u "${LOCAL_UID}" "${LOCAL_USER}" || echo "Warning: cannot change UID for ${LOCAL_USER}, leaving as ${current_uid}" >&2 + else + echo "Warning: target UID ${LOCAL_UID} already in use; leaving user ${LOCAL_USER} with UID ${current_uid}" >&2 + fi + fi + else + # Create the user only if the target UID is free + if getent passwd "${LOCAL_UID}" >/dev/null 2>&1; then + # Someone owns the UID, pick that user + existing_user=$(getent passwd "${LOCAL_UID}" | cut -d: -f1) + echo "Host UID ${LOCAL_UID} in use by '${existing_user}'; using that user inside container." + LOCAL_USER="${existing_user}" + else + useradd -m -u "${LOCAL_UID}" -g "${LOCAL_GID}" -s /bin/bash "${LOCAL_USER}" || { + echo "Warning: failed to create user ${LOCAL_USER} with UID ${LOCAL_UID}; falling back to root inside container." >&2 + LOCAL_USER=root + } + fi + fi + + # Ensure home dir exists and is owned by requested uid/gid where possible + mkdir -p "/home/${LOCAL_USER}" + chown -R "${LOCAL_UID}:${LOCAL_GID}" "/home/${LOCAL_USER}" 2>/dev/null || true +} + +setup_sudo() { + if [[ "${ENABLE_CONTAINER_SUDO}" == "1" ]]; then + usermod -aG sudo "${LOCAL_USER}" || true + echo "${LOCAL_USER} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/90-local-user + chmod 0440 /etc/sudoers.d/90-local-user + else + rm -f /etc/sudoers.d/90-local-user + fi +} + +fix_mount_ownership() { + mkdir -p "/home/${LOCAL_USER}/.ssh" + chown -R "${LOCAL_UID}:${LOCAL_GID}" "/home/${LOCAL_USER}/.ssh" 2>/dev/null || true + chown "${LOCAL_UID}:${LOCAL_GID}" "/home/${LOCAL_USER}/.gitconfig" 2>/dev/null || true + chown "${LOCAL_UID}:${LOCAL_GID}" "/home/${LOCAL_USER}/.bashrc" 2>/dev/null || true + chown "${LOCAL_UID}:${LOCAL_GID}" "/home/${LOCAL_USER}/.bash_aliases" 2>/dev/null || true +} + +ensure_group >/dev/null +ensure_user +setup_sudo +fix_mount_ownership + +export HOME="/home/${LOCAL_USER}" +cd "${WORKDIR_IN_CONTAINER}" + +exec runuser -u "${LOCAL_USER}" -- "$@" +DOCKER_SCRIPT diff --git a/app/run_migration.py b/app/run_migration.py new file mode 100644 index 000000000..05c5dcfeb --- /dev/null +++ b/app/run_migration.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""CLI entry point for the Qt5→Qt6 migration pipeline. + +Usage examples +-------------- +# Run the Q1 pilot batch on both tracks: +python run_migration.py \\ + --batch-id q1_pilot \\ + --file-list ./q1_files.txt \\ + --repo-root . \\ + --source-root ./src + +# Dry-run: show files and chunk info without migrating: +python run_migration.py --batch-id q1_pilot --file-list ./q1_files.txt --dry-run +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import sys +from pathlib import Path + + +def _configure_logging(verbose: bool) -> None: + level = logging.DEBUG if verbose else logging.INFO + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)-8s %(name)s – %(message)s", + datefmt="%H:%M:%S", + ) + + +def _dry_run(file_list_path: Path, source_root: Path) -> None: + """Print file list with chunk estimates without calling any API.""" + from migration_pipeline.chunker import split_semantic + from migration_pipeline.config import CHUNK_SIZE_LINES + + files = _load_plain_list(file_list_path, source_root) + print(f"\nDry-run: {len(files)} files in {file_list_path.name}\n") + print(f"{'#':<4} {'File':<60} {'Lines':>6} {'Chunks':>6}") + print("-" * 82) + for i, fp in enumerate(files, 1): + p = Path(fp) + if not p.exists(): + print(f"{i:<4} {fp:<60} {'NOT FOUND':>6}") + continue + code = p.read_text(errors="ignore") + lines = len(code.splitlines()) + chunks = split_semantic(code, CHUNK_SIZE_LINES) + print(f"{i:<4} {str(p.name):<60} {lines:>6} {len(chunks):>6}") + print() + + +def _load_plain_list(list_path: Path, source_root: Path) -> list[str]: + base = source_root.resolve() + result = [] + for raw in list_path.read_text().splitlines(): + s = raw.strip() + if not s or s.startswith("#"): + continue + p = Path(s) + result.append(str((base / p).resolve() if not p.is_absolute() else p)) + return result + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Qt5→Qt6 async migration pipeline", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "--batch-id", + required=True, + help="Unique identifier for this batch (e.g. q1_pilot)", + ) + parser.add_argument( + "--file-list", + required=True, + type=Path, + help="Path to the prioritised file list (one path per line)", + ) + parser.add_argument( + "--repo-root", + default=".", + type=Path, + help="Root of the git repository (default: current directory)", + ) + parser.add_argument( + "--source-root", + default=".", + type=Path, + help="Root of the C++ source tree (default: current directory)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print file list and chunk estimates; do not call any API", + ) + parser.add_argument( + "--model", + choices=("glm", "qwen"), + default="glm", + help="Model family for API calls (default: glm)", + ) + parser.add_argument( + "--verbose", "-v", + action="store_true", + help="Enable DEBUG logging", + ) + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + _configure_logging(args.verbose) + + if not args.file_list.exists(): + print(f"ERROR: File list not found: {args.file_list}", file=sys.stderr) + sys.exit(1) + + if args.dry_run: + _dry_run(args.file_list, args.source_root.resolve()) + return + + from migration_pipeline.api_client import set_model_family + from migration_pipeline.orchestrator import run_batch + + set_model_family(args.model) + asyncio.run( + run_batch( + batch_id=args.batch_id, + file_list_path=args.file_list, + repo_root=args.repo_root.resolve(), + source_root=args.source_root.resolve(), + ) + ) + + +if __name__ == "__main__": + main() diff --git a/app/tests/test_api_client.py b/app/tests/test_api_client.py new file mode 100644 index 000000000..ec0afa3fe --- /dev/null +++ b/app/tests/test_api_client.py @@ -0,0 +1,32 @@ +import importlib +import os +import unittest + + +def _set_required_env() -> None: + os.environ["SCW_ACCESS_KEY"] = "key0" + os.environ["SCW_SECRET_KEY"] = "key1" + os.environ["SCW_DEFAULT_ORGANIZATION_ID"] = "key2" + os.environ["SCW_DEFAULT_PROJECT_ID"] = "key3" + os.environ["SCW_SERVERLESS"] = "key4" + + +class ApiClientModelSelectionTests(unittest.TestCase): + def setUp(self) -> None: + _set_required_env() + import migration_pipeline.api_client as api_client + + self.api_client = importlib.reload(api_client) + + def test_default_model_family_is_glm(self) -> None: + self.assertEqual(self.api_client.get_selected_model_family(), "glm") + self.assertEqual(self.api_client.select_model(10), "glm-5.2") + + def test_switch_to_qwen_model(self) -> None: + self.api_client.set_model_family("qwen") + self.assertEqual(self.api_client.select_model(10), "qwen3.5-397b-a17b") + + def test_invalid_model_family_raises(self) -> None: + with self.assertRaises(ValueError): + self.api_client.set_model_family("invalid") + diff --git a/app/tests/test_config.py b/app/tests/test_config.py new file mode 100644 index 000000000..7c1dc1626 --- /dev/null +++ b/app/tests/test_config.py @@ -0,0 +1,58 @@ +import importlib +import os +import sys +import tempfile +import unittest +from pathlib import Path + + +class ConfigEnvLoadingTests(unittest.TestCase): + def setUp(self) -> None: + self._old_cwd = Path.cwd() + self._keys = [ + "SCW_ACCESS_KEY", + "SCW_SECRET_KEY", + "SCW_DEFAULT_ORGANIZATION_ID", + "SCW_DEFAULT_PROJECT_ID", + "SCW_SERVERLESS", + ] + self._backup = {k: os.environ.get(k) for k in self._keys} + + def tearDown(self) -> None: + os.chdir(self._old_cwd) + for key, value in self._backup.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + def _clear_env(self) -> None: + for key in self._keys: + os.environ.pop(key, None) + + def test_import_uses_dotenv_fallback(self) -> None: + self._clear_env() + with tempfile.TemporaryDirectory() as tmp: + os.chdir(tmp) + Path(".env").write_text( + "SCW_ACCESS_KEY=key0\n" + "SCW_SECRET_KEY=key1\n" + "SCW_DEFAULT_ORGANIZATION_ID=key2\n" + "SCW_DEFAULT_PROJECT_ID=key3\n" + "SCW_SERVERLESS=key4\n" + ) + sys.modules.pop("migration_pipeline.config", None) + config = importlib.import_module("migration_pipeline.config") + self.assertEqual(config.SCW_SERVERLESS, "key4") + self.assertEqual( + config.BASE_URL, + "https://api.scaleway.ai/key4/v1/chat/completions", + ) + + def test_import_fails_when_required_env_missing(self) -> None: + self._clear_env() + with tempfile.TemporaryDirectory() as tmp: + os.chdir(tmp) + sys.modules.pop("migration_pipeline.config", None) + with self.assertRaisesRegex(RuntimeError, "SCW_ACCESS_KEY"): + importlib.import_module("migration_pipeline.config") diff --git a/app/tests/test_run_migration.py b/app/tests/test_run_migration.py new file mode 100644 index 000000000..fd8ad990c --- /dev/null +++ b/app/tests/test_run_migration.py @@ -0,0 +1,34 @@ +import sys +import unittest +from unittest.mock import patch + +import run_migration + + +class RunMigrationCliTests(unittest.TestCase): + def test_model_defaults_to_glm(self) -> None: + argv = [ + "run_migration.py", + "--batch-id", + "b1", + "--file-list", + "files.txt", + ] + with patch.object(sys, "argv", argv): + args = run_migration._parse_args() + self.assertEqual(args.model, "glm") + + def test_model_accepts_qwen(self) -> None: + argv = [ + "run_migration.py", + "--batch-id", + "b1", + "--file-list", + "files.txt", + "--model", + "qwen", + ] + with patch.object(sys, "argv", argv): + args = run_migration._parse_args() + self.assertEqual(args.model, "qwen") +