diff --git a/.gitignore b/.gitignore index 06a4b4bfe..4d591b4b4 100644 --- a/.gitignore +++ b/.gitignore @@ -88,4 +88,5 @@ fix_*.py rpms/ deb_stage packages-out +external-dependencies/offline-cache/ diff --git a/external-dependencies/offline-cache/README.md b/external-dependencies/offline-cache/README.md new file mode 100644 index 000000000..b6e2e13d5 --- /dev/null +++ b/external-dependencies/offline-cache/README.md @@ -0,0 +1,238 @@ +# Offline Package Cache for Air-Gapped Environment + +> **Important:** This directory is excluded from version control (`.gitignore`). The offline cache must be built separately on a machine with internet access, then transferred to the air-gapped environment via secure media or artifact repository. + +This directory contains all external dependencies required for building the Legatus project in an air-gapped (offline) environment. + +## Directory Structure + +``` +offline-cache/ +├── qt6/ # Qt 6.10.2 installation (2.2GB, 12,320 files) +│ └── 6.10.2/ +│ └── gcc_64/ +│ └── lib/cmake/Qt6/ +│ +├── src/ # Source tarballs for custom builds +│ ├── KDReports-2.3.0.tar.gz (8.3MB) +│ ├── Log4Qt-2.0.0.tar.gz (1.0MB) +│ └── grantlee-20d415b.tar.gz (2.5MB) +│ +├── deb/ # Ubuntu 26.04 development packages (49 packages, 51MB) +│ ├── build-essential_*.deb +│ ├── cmake_*.deb +│ ├── libqt6*.deb +│ ├── libboost-dev_*.deb +│ ├── libssl-dev_*.deb +│ └── ... (46 more) +│ +├── scripts/ # Offline installation scripts +│ ├── install-offline.sh # Main installation script +│ └── package-list.txt # Complete package inventory +│ +└── README.md # This file +``` + +## Contents Summary + +| Component | Size | Files | Description | +|-----------|------|-------|-------------| +| Qt 6.10.2 | 2.2GB | 12,320 | Complete Qt installation (gcc_64) | +| Source Tarballs | 12MB | 3 | KDReports, Log4Qt, Grantlee | +| Ubuntu .deb | 51MB | 49 | Development packages | +| **Total** | **2.3GB** | **12,372** | Complete offline cache | + +## Usage in Air-Gapped Environment + +### 1. Copy to Air-Gapped System + +Transfer the entire `offline-cache/` directory to your air-gapped build system: + +```bash +# On connected system +tar -czf offline-cache.tar.gz offline-cache/ + +# Transfer via secure medium (USB, secure transfer, etc.) + +# On air-gapped system +tar -xzf offline-cache.tar.gz +``` + +### 2. Install Qt 6.10.2 + +```bash +# Copy Qt installation to target location +cp -r offline-cache/qt6/6.10.2 /opt/Qt/ + +# Set environment variables +export Qt6_DIR=/opt/Qt/6.10.2/gcc_64 +export PATH=$Qt6_DIR/bin:$PATH +``` + +### 3. Install Ubuntu Development Packages + +```bash +# Install all .deb packages +cd offline-cache/deb +sudo dpkg -i *.deb + +# Or use apt with local repository +cd offline-cache/deb +sudo apt install ./*.deb +``` + +### 4. Build Custom Dependencies + +```bash +# KDReports +cd /tmp +tar -xzf offline-cache/src/KDReports-2.3.0.tar.gz +cd KDReports-2.3.0 +mkdir build && cd build +cmake -DCMAKE_PREFIX_PATH=/opt/Qt/6.10.2/gcc_64 .. +make -j8 +sudo make install + +# Log4Qt +cd /tmp +tar -xzf offline-cache/src/Log4Qt-2.0.0.tar.gz +cd Log4Qt-2.0.0 +mkdir build && cd build +cmake -DCMAKE_PREFIX_PATH=/opt/Qt/6.10.2/gcc_64 .. +make -j8 +sudo make install + +# Grantlee +cd /tmp +tar -xzf offline-cache/src/grantlee-20d415b.tar.gz +cd grantlee-20d415b +mkdir build && cd build +cmake -DGRANTLEE_BUILD_WITH_QT6=ON -DCMAKE_PREFIX_PATH=/opt/Qt/6.10.2/gcc_64 .. +make -j8 +sudo make install +``` + +### 5. Verify Installation + +```bash +# Check Qt version +qmake6 --version + +# Check CMake +cmake --version + +# Check compilers +g++ --version +clang++ --version +``` + +## Package Inventory + +### Qt Components (6.10.2) +- Qt Base (Core, Gui, Widgets, Network, etc.) +- Qt SVG +- Qt Concurrent +- Qt Test +- Qt Linguist Tools + +### Ubuntu Development Packages (49 total) +- **Build Tools:** build-essential, cmake, clang, gcc, g++ +- **Qt6 Libraries:** libqt6core6, libqt6gui6, libqt6widgets6, libqt6svg6, etc. +- **Development Headers:** libqt6-dev, libqt6svg6-dev, libkf6texttemplate-dev +- **Database:** libpq-dev, postgresql-server-dev-all +- **Testing:** libgtest-dev, libgmock-dev +- **Security:** libssl-dev, libbotan-3-dev +- **Multimedia:** libvlc-dev, vlc-plugin-base +- **Other:** libboost-dev, libgeographiclib-dev, libpoppler-qt6-dev, libxml2-dev, libicu-dev + +### Source Tarballs +- **KDReports 2.3.0:** Report generation library +- **Log4Qt 2.0.0:** Logging framework +- **Grantlee (20d415b):** String template library + +## Jenkins Integration + +The offline cache is used by Jenkins pipelines in air-gapped environments: + +```groovy +// In Jenkinsfile +stage('Setup Offline Dependencies') { + steps { + script { + // Copy offline cache to build workspace + sh 'cp -r /path/to/offline-cache ${WORKSPACE}/' + + // Install Qt + sh 'cp -r ${WORKSPACE}/offline-cache/qt6/6.10.2 /opt/Qt/' + + // Install .deb packages + sh 'cd ${WORKSPACE}/offline-cache/deb && sudo dpkg -i *.deb' + + // Build custom dependencies + sh 'bash ${WORKSPACE}/offline-cache/scripts/install-offline.sh' + } + } +} +``` + +## Maintenance + +### Updating Dependencies + +To update the offline cache: + +1. Run the download script on a connected system: + ```bash + cd external-dependencies + bash scripts/download-all-dependencies.sh ./offline-cache + ``` + +2. Verify the downloaded packages: + ```bash + bash scripts/verify-offline-cache.sh ./offline-cache + ``` + +3. Transfer to air-gapped system + +### Verification Checklist + +- [ ] Qt 6.10.2 installation complete (12,320+ files) +- [ ] All source tarballs present (KDReports, Log4Qt, Grantlee) +- [ ] All 49 .deb packages downloaded +- [ ] Package list matches inventory +- [ ] Installation scripts tested + +## Troubleshooting + +### Qt Installation Issues +- Ensure correct architecture (gcc_64 for x86_64) +- Check CMAKE_PREFIX_PATH includes Qt directory +- Verify Qt6_DIR environment variable + +### Package Installation Issues +- Check Ubuntu version matches (26.04) +- Ensure all dependencies are satisfied +- Use `dpkg --configure -a` to fix broken installs + +### Build Issues +- Verify all custom dependencies are installed +- Check CMake can find Qt6 components +- Ensure compiler toolchain is complete + +## License Compliance + +All packages include their respective licenses: +- Qt 6.10.2: LGPL v3 / Commercial +- KDReports: Apache 2.0 +- Log4Qt: Apache 2.0 +- Grantlee: BSD 2-Clause +- Ubuntu packages: Various (see individual .deb files) + +SBOM (Software Bill of Materials) is generated by the CI/CD pipeline using `generate-sbom.sh`. + +## Contact + +For questions or issues with the offline cache, refer to: +- `external-dependencies/DEPENDENCIES-INVENTORY.md` - Complete dependency list +- `external-dependencies/AIRGAP-SUMMARY.md` - Air-gap configuration +- Jenkins pipeline documentation \ No newline at end of file diff --git a/external-dependencies/scripts/download-all-dependencies.sh b/external-dependencies/scripts/download-all-dependencies.sh new file mode 100755 index 000000000..ffba3e87f --- /dev/null +++ b/external-dependencies/scripts/download-all-dependencies.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# +# Download all external dependencies for air-gapped environment +# Creates offline package cache for Jenkins/Dockerfile use +# +# Usage: ./download-all-dependencies.sh [output-directory] +# +# Output directory structure: +# offline-cache/ +# ├── qt6/ # Qt6 packages from aqtinstall +# ├── src/ # Source tarballs (KDReports, Log4Qt, Grantlee) +# ├── deb/ # Ubuntu .deb packages +# └── scripts/ # Installation scripts for offline use +# + +set -euo pipefail + +# Configuration +OUTPUT_DIR="${1:-./offline-cache}" +QT_VERSION="6.10.2" +QT_MODULES="qtmultimedia qtserialport qtimageformats qtwebsockets qtwebengine qtpdf qtwebview qt5compat qtwebchannel qtpositioning qtlocation qtsensors" + +# Air-gapped environment configuration (from Dockerfiles) +NEXUS_REPO="http://remote-repo.sl.defence-elec.de:8081" +PIP_INDEX_URL="${NEXUS_REPO}/repository/proxypipy/" +APT_SOURCES="${NEXUS_REPO}/repository/ubuntu" + +echo "=========================================" +echo "Downloading External Dependencies" +echo "=========================================" +echo "Output directory: ${OUTPUT_DIR}" +echo "Qt Version: ${QT_VERSION}" +echo "=========================================" + +# Create directory structure +mkdir -p "${OUTPUT_DIR}"/{qt6,src,deb,scripts} + +# Run download in Docker container with proper environment +echo "" +echo "Running download in Ubuntu 26.04 Docker container..." +echo "" + +docker run --rm \ + -v "$(pwd)/${OUTPUT_DIR}:/cache" \ + -e PIP_INDEX_URL="${PIP_INDEX_URL}" \ + -e NEXUS_REPO="${NEXUS_REPO}" \ + ubuntu:26.04 \ + bash -c " +set -euo pipefail + +OUTPUT_DIR='/cache' +QT_VERSION='${QT_VERSION}' +QT_MODULES='${QT_MODULES}' +NEXUS_REPO='${NEXUS_REPO}' +PIP_INDEX_URL='${PIP_INDEX_URL}' + +echo '=== Installing required packages ===' +apt-get update -qq +apt-get install -y -qq python3 python3-venv python3-pip git wget curl apt-transport-https + +echo '' +echo '=== Setting up Python virtual environment for aqtinstall ===' +python3 -m venv /tmp/aqtinstall_venv +source /tmp/aqtinstall_venv/bin/activate + +# Install aqtinstall from standard PyPI (not available in internal proxy) +echo 'Installing aqtinstall from standard PyPI...' +pip install --quiet --index-url https://pypi.org/simple aqtinstall + +echo '' +echo '=== Downloading Qt6 \${QT_VERSION} ===' +aqt install-qt linux desktop \${QT_VERSION} linux_gcc_64 \\ + --modules \${QT_MODULES} \\ + --outputdir /cache/qt6 \\ + --base \${NEXUS_REPO}/repository/qt-mirror/ + +echo '' +echo 'Qt6 downloaded to /cache/qt6' + +echo '' +echo '=== Downloading Source Tarballs ===' + +# KDReports +echo 'Downloading KDReports 2.3.0...' +git clone --depth 1 --branch kdreports-2.3.0 \\ + https://github.com/KDAB/KDReports.git \\ + /tmp/KDReports +tar -czf /cache/src/KDReports-2.3.0.tar.gz -C /tmp KDReports +rm -rf /tmp/KDReports + +# Log4Qt +echo 'Downloading Log4Qt 2.0.0...' +git clone --depth 1 --branch v2.0.0 \\ + https://github.com/MEONMedical/Log4Qt.git \\ + /tmp/Log4Qt +tar -czf /cache/src/Log4Qt-2.0.0.tar.gz -C /tmp Log4Qt +rm -rf /tmp/Log4Qt + +# Grantlee +echo 'Downloading Grantlee (master branch)...' +git clone --depth 1 \\ + https://github.com/steveire/grantlee.git \\ + /tmp/grantlee +GRANTLEE_COMMIT=\$(cd /tmp/grantlee && git rev-parse HEAD) +echo 'Grantlee commit: \${GRANTLEE_COMMIT}' +tar -czf /cache/src/grantlee-\${GRANTLEE_COMMIT:0:7}.tar.gz -C /tmp grantlee +rm -rf /tmp/grantlee + +echo '' +echo 'Source tarballs downloaded to /cache/src' + +# Cleanup +deactivate 2>/dev/null || true +rm -rf /tmp/aqtinstall_venv + +echo '' +echo '=== Download Complete ===' +" \ No newline at end of file diff --git a/external-dependencies/scripts/download-deb-packages.sh b/external-dependencies/scripts/download-deb-packages.sh new file mode 100755 index 000000000..4c9064b95 --- /dev/null +++ b/external-dependencies/scripts/download-deb-packages.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# +# Download Ubuntu .deb packages for offline use +# Run this script inside Ubuntu 26.04 container +# +# Usage: ./download-deb-packages.sh [output-directory] +# + +set -euo pipefail + +OUTPUT_DIR="${1:-./offline-cache/deb}" + +echo "=========================================" +echo "Downloading Ubuntu .deb Packages" +echo "=========================================" +echo "Output directory: ${OUTPUT_DIR}" +echo "=========================================" + +# Run download in Docker container with proper environment +echo "" +echo "Running download in Ubuntu 26.04 Docker container..." +echo "" + +docker run --rm \ + -v "$(pwd)/${OUTPUT_DIR}:/cache/deb" \ + ubuntu:26.04 \ + bash -c " +set -euo pipefail + +OUTPUT_DIR='/cache/deb' + +# List of all required packages +PACKAGES=( + # Build tools + build-essential + cmake + ninja-build + clang + gnupg + curl + jq + openssh-client + pkg-config + unzip + + # Qt6 libraries + libqt6svg6-dev + libkf6texttemplate-dev + libqt6core6 + libqt6gui6 + libqt6widgets6 + libqt6sql6 + libqt6sql6-psql + libqt6test6 + qt6-base-dev + qt6-l10n-tools + qt6-serialport-dev + qt6-svg-dev + qt6-tools-dev-tools + qt6-tools-dev + qt6-websockets-dev + + # Other libraries + libgl1-mesa-dev + libxkbcommon-x11-0 + libfontconfig1 + libdbus-1-3 + libpulse0 + libxcb-glx0 + libxcb-cursor0 + libbz2-dev + libpoppler-dev + libpoppler-qt6-dev + libboost-dev + libgeographiclib-dev + libbotan-3-dev + libssl-dev + libxml2-dev + libicu-dev + + # Testing + libgtest-dev + libgmock-dev + + # Database + libpq-dev + postgresql-server-dev-all + + # Multimedia + libvlc-dev + vlc-plugin-base + + # Python + python3 + python3-venv + python3-pip + + # Packaging + rpm +) + +mkdir -p \"\${OUTPUT_DIR}\" + +echo 'Updating package lists...' +apt-get update -qq + +echo \"Downloading \${#PACKAGES[@]} packages...\" + +# Download all packages with dependencies +for pkg in \"\${PACKAGES[@]}\"; do + echo \" Downloading \${pkg}...\" + apt-get download \"\${pkg}\" || echo \"Warning: Could not download \${pkg}\" +done + +# Move all .deb files to output directory +echo 'Collecting .deb packages...' +find / -name \"*.deb\" -type f 2>/dev/null | head -1000 | xargs -I {} cp {} \"\${OUTPUT_DIR}/\" || true + +DEB_COUNT=\$(ls -1 \"\${OUTPUT_DIR}\"/*.deb 2>/dev/null | wc -l || echo 0) +echo '' +echo '=========================================' +echo 'Download Complete!' +echo '=========================================' +echo \"Downloaded \${DEB_COUNT} .deb packages to \${OUTPUT_DIR}\" +echo '=========================================' + +# Create package list for reference +cat > /cache/package-list.txt << EOF +# Ubuntu 26.04 Package List +# Generated: \$(date -u +\"%Y-%m-%dT%H:%M:%SZ\") +# Total packages: \${DEB_COUNT} + +\$(ls -1 \"\${OUTPUT_DIR}\"/*.deb 2>/dev/null | xargs -n1 basename 2>/dev/null | sort || echo 'No packages downloaded') +EOF + +echo 'Package list created: /cache/package-list.txt' +" \ No newline at end of file diff --git a/external-dependencies/scripts/verify-offline-cache.sh b/external-dependencies/scripts/verify-offline-cache.sh new file mode 100755 index 000000000..69ebb392a --- /dev/null +++ b/external-dependencies/scripts/verify-offline-cache.sh @@ -0,0 +1,78 @@ +#!/bin/bash +# Verify Offline Cache Integrity +# Usage: bash verify-offline-cache.sh [offline-cache-path] + +set -euo pipefail + +OFFLINE_CACHE="${1:-./offline-cache}" + +echo "=== Verifying Offline Cache ===" +echo "Path: $OFFLINE_CACHE" +echo "" + +ERRORS=0 + +# Check Qt installation +echo "Checking Qt 6.10.2..." +if [ -d "$OFFLINE_CACHE/qt6/6.10.2/gcc_64" ]; then + QT_FILES=$(find "$OFFLINE_CACHE/qt6/6.10.2/gcc_64" -type f | wc -l) + if [ "$QT_FILES" -ge 12000 ]; then + echo "✓ Qt installation complete ($QT_FILES files)" + else + echo "✗ Qt installation incomplete (expected 12000+, found $QT_FILES)" + ERRORS=$((ERRORS + 1)) + fi +else + echo "✗ Qt installation directory missing" + ERRORS=$((ERRORS + 1)) +fi + +# Check source tarballs +echo "" +echo "Checking source tarballs..." +for tarball in KDReports-2.3.0.tar.gz Log4Qt-2.0.0.tar.gz grantlee-20d415b.tar.gz; do + if [ -f "$OFFLINE_CACHE/src/$tarball" ]; then + SIZE=$(du -h "$OFFLINE_CACHE/src/$tarball" | cut -f1) + echo "✓ $tarball ($SIZE)" + else + echo "✗ $tarball missing" + ERRORS=$((ERRORS + 1)) + fi +done + +# Check .deb packages +echo "" +echo "Checking Ubuntu .deb packages..." +if [ -d "$OFFLINE_CACHE/deb" ]; then + DEB_COUNT=$(ls -1 "$OFFLINE_CACHE/deb"/*.deb 2>/dev/null | wc -l) + if [ "$DEB_COUNT" -ge 49 ]; then + echo "✓ Debian packages complete ($DEB_COUNT packages)" + else + echo "✗ Debian packages incomplete (expected 49+, found $DEB_COUNT)" + ERRORS=$((ERRORS + 1)) + fi +else + echo "✗ Debian packages directory missing" + ERRORS=$((ERRORS + 1)) +fi + +# Check package list +echo "" +echo "Checking package inventory..." +if [ -f "$OFFLINE_CACHE/scripts/package-list.txt" ]; then + echo "✓ Package list present" +else + echo "✗ Package list missing" + ERRORS=$((ERRORS + 1)) +fi + +# Summary +echo "" +echo "=== Verification Summary ===" +if [ "$ERRORS" -eq 0 ]; then + echo "✓ All checks passed" + exit 0 +else + echo "✗ $ERRORS check(s) failed" + exit 1 +fi \ No newline at end of file