#!/usr/bin/python3 # -*- coding: utf-8 -*- """ DEB package builder for Ubuntu 26.04 - Qt6 Release Creates 5 separate DEB packages as per customer requirements: 1. qt6-release-runtime - binaries and libraries from S 2. qt6-release-sources - Qt6 C++ sources from S 3. qt6-release-ctest-runtime - CTest executables 4. qt6-release-ctest-sources - CTest source files 5. qt6-release-build-pipeline - Dockerfiles, build scripts, CI/CD files """ import argparse import fnmatch import os import shutil import subprocess import sys from email.utils import formatdate SKIP_DIRS = {'build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'} SOURCE_EXTENSIONS = ('.cpp', '.h', '.hpp', '.cc', '.cxx', '.hxx') PIPELINE_FILES = [ "new_docker_build.sh", "Dockerfile", "jenkins-ci-rme/Jenkinsfile", "jenkins-ci-rme/jenkins/Jenkinsfile", "jenkins-ci-rme/jenkins_builder_ubuntu26.sh", "jenkins-ci-rme/jenkins_publisher_ubuntu26.sh", "jenkins-ci-rme/release-16-10-2026.sh", "rpmbuild/make_rpm_ubuntu26.py", "CMakeLists.txt", ] def verbose(): verb = os.getenv("EMERGE_VERBOSE") if verb is not None and verb.isdigit() and int(verb) > 0: return int(verb) return 0 def debug(message): if verbose() >= 1: print(f"debug: {message}") return True def info(message): print(f"info: {message}") return True def warning(message): print(f"warning: {message}") return True def error(message): die(message) def die(message): print(f"emerge fatal error: {message}", file=sys.stderr) sys.exit(1) def run(cmd, label, env=None): """Run a shell command, print its output and die on failure.""" debug(f"Executing: {cmd}") ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) if ret.returncode != 0: print(ret.stdout) print(ret.stderr) die(f"{label} with code {ret.returncode}") def remove_directory(path): if os.path.exists(path): shutil.rmtree(path) def is_dds_file(filepath): """True if the path is DDS-related and must be excluded.""" return 'dds' in filepath.lower() def prune_dirs(dirs, root, file_filter): """Prune hidden/build dirs in-place (and dirs matching file_filter).""" dirs[:] = [d for d in dirs if not d.startswith('.') and d not in SKIP_DIRS and (file_filter is None or not file_filter(os.path.join(root, d)))] def copy_src_dir_to_dest_dir(src_dir, dest_dir, file_filter=None): """Copy the source tree, skipping hidden/build dirs and filtered files.""" os.makedirs(dest_dir, exist_ok=True) for root, dirs, files in os.walk(src_dir): prune_dirs(dirs, root, file_filter) for file in files: src_file = os.path.join(root, file) if file_filter and file_filter(src_file): continue dest_file = os.path.join(dest_dir, os.path.relpath(src_file, src_dir)) os.makedirs(os.path.dirname(dest_file), exist_ok=True) shutil.copy2(src_file, dest_file) def find_test_module_source(pkg_dir, test_name, release_dirs): """Module dir (relative to pkg_dir) whose CMakeLists defines test_name, or None.""" for rel_dir in release_dirs: module_dir = os.path.join(pkg_dir, rel_dir) if not os.path.exists(module_dir): continue for root, dirs, files in os.walk(module_dir): prune_dirs(dirs, root, None) if 'CMakeLists.txt' not in files: continue try: with open(os.path.join(root, 'CMakeLists.txt')) as f: content = f.read() except Exception: continue if f'ADD_EXECUTABLE({test_name}' in content or f'ADD_TEST(NAME {test_name}' in content: return os.path.relpath(root, pkg_dir) return None def find_test_executables(pkg_dir, release_dirs): """(executable, module) pairs for the test executables in build4/bin/.""" test_executables = [] build_bin_dir = os.path.join(pkg_dir, "build4", "bin") if not os.path.exists(build_bin_dir): warning(f"Build bin directory not found: {build_bin_dir}") return test_executables for filename in os.listdir(build_bin_dir): file_path = os.path.join(build_bin_dir, filename) if not (os.path.isfile(file_path) and os.access(file_path, os.X_OK)): continue if 'test' not in filename.lower(): continue module_path = find_test_module_source(pkg_dir, filename, release_dirs) if module_path: test_executables.append((file_path, module_path)) debug(f" Found test executable: {filename} from {module_path}") else: debug(f" Skipping test executable (no module found): {filename}") return test_executables def find_test_sources(pkg_dir, release_dirs): """Relative paths of test source files (name containing 'test', .cpp/.h).""" test_sources = [] for rel_dir in release_dirs: module_dir = os.path.join(pkg_dir, rel_dir) if not os.path.exists(module_dir): continue for root, dirs, files in os.walk(module_dir): prune_dirs(dirs, root, None) for filename in files: if 'test' in filename.lower() and filename.endswith(('.cpp', '.h')): file_path = os.path.join(root, filename) rel_path = os.path.relpath(file_path, pkg_dir) test_sources.append(rel_path) debug(f" Found test source: {rel_path}") return test_sources def write_deb_file(deb_dir, name, content, executable=False): """Write a debian/* file and log its creation.""" path = os.path.join(deb_dir, name) with open(path, "w") as f: f.write(content) if executable: os.chmod(path, 0o755) info(f"Created {path}") def create_debian_control(deb_dir, package_name, version, build_number, description, depends="", section="devel"): write_deb_file(deb_dir, "control", f"""Source: {package_name} Section: {section} Priority: optional Maintainer: Legatus Team Build-Depends: debhelper (>= 13), cmake, ninja-build, qt6-base-dev, qt6-tools-dev Standards-Version: 4.6.0 Package: {package_name} Architecture: any Depends: ${{shlibs:Depends}}, ${{misc:Depends}}{", " + depends if depends else ""} Description: {description} Legatus Qt6 Release Package This package is part of the Legatus Qt6 migration release. """) def create_debian_changelog(deb_dir, package_name, version, build_number): write_deb_file(deb_dir, "changelog", f"""{package_name} ({version}-{build_number}) unstable; urgency=medium * Qt6 Release Build -- Legatus Team {formatdate(localtime=True)} """) def create_debian_rules(deb_dir, cmake_flags="", skip_tests=False): test_override = """ override_dh_auto_test: # Tests skipped for this package type """ if skip_tests else "" write_deb_file(deb_dir, "rules", f"""#!/usr/bin/make -f %: dh $@ --buildsystem=cmake override_dh_auto_configure: # -GNinja: required, see build_all_debs (CMake 4.x Makefiles generator # breaks AUTOMOC; the cmake buildsystem would otherwise use make). dh_auto_configure -- -GNinja -DCMAKE_BUILD_TYPE=Release {cmake_flags} override_dh_auto_build: dh_auto_build --parallel override_dh_auto_install: dh_auto_install --destdir=debian/tmp {test_override}""", executable=True) def create_debian_rules_nobuild(deb_dir, skip_shlibdeps=False): content = """#!/usr/bin/make -f %: dh $@ override_dh_auto_build: # No build step needed - pre-built files only override_dh_auto_configure: # No configure step needed - pre-built files only override_dh_auto_install: dh_auto_install """ if skip_shlibdeps: content += """ override_dh_shlibdeps: # Skip shlibdeps for test executables - Qt6 deps are external system packages """ write_deb_file(deb_dir, "rules", content, executable=True) def create_debian_install(deb_dir, install_lines): install_file = os.path.join(deb_dir, "install") with open(install_file, "w") as f: for line in install_lines: f.write(line + "\n") info(f"Created {install_file} with {len(install_lines)} entries") def create_manifest(output_dir, package_name, pkg_dir): """Write the manifest of package contents (from the DEB, or the pkg dir).""" manifest_file = os.path.join(output_dir, "manifests", f"{package_name}.manifest") os.makedirs(os.path.dirname(manifest_file), exist_ok=True) deb_file = next((os.path.join(output_dir, f) for f in os.listdir(output_dir) if f.startswith(package_name) and f.endswith(".deb")), None) manifest_lines = [] if deb_file and os.path.exists(deb_file): result = subprocess.run(["dpkg-deb", "--contents", deb_file], capture_output=True, text=True) if result.returncode == 0: # dpkg-deb --contents: "-rw-r--r-- root/root 1234 2024-01-01 00:00 ./path" for line in result.stdout.strip().split("\n"): parts = line.split() if len(parts) >= 6: file_path = parts[-1] if file_path.startswith("./"): file_path = file_path[2:] elif file_path.startswith("/"): file_path = file_path[1:] if file_path: manifest_lines.append(file_path) elif os.path.exists(pkg_dir): info(f"Warning: DEB file not found for {package_name}, scanning directory instead") for root, dirs, files in os.walk(pkg_dir): for file in files: manifest_lines.append(os.path.relpath(os.path.join(root, file), pkg_dir)) manifest_lines.sort() with open(manifest_file, "w") as f: for line in manifest_lines: f.write(line + "\n") info(f"Created manifest: {manifest_file} ({len(manifest_lines)} entries)") return manifest_lines def validate_package_contents(package_name, manifest_lines, allowed_patterns, forbidden_patterns=None): """Validate package contents against the allowlist/forbidden list.""" errors = [] warnings = [] for file_path in manifest_lines: if file_path.endswith('/'): continue if forbidden_patterns: for pattern in forbidden_patterns: if fnmatch.fnmatch(file_path, pattern): errors.append(f"FORBIDDEN file in {package_name}: {file_path} (matches {pattern})") allowed = bool(allowed_patterns) and any(fnmatch.fnmatch(file_path, p) for p in allowed_patterns) if not allowed and allowed_patterns: warnings.append(f"Unexpected file in {package_name}: {file_path}") for err in errors: error(f"Validation error: {err}") if errors: return False for warn in warnings: warning(f"Validation warning: {warn}") return True def build_deb(pkg_name, pkg_dir, output_dir, version, build_number, cmake_flags="", nobuild=False, skip_tests=False, skip_shlibdeps=False, install_lines=None, description="", section="devel", depends=""): """Build one DEB package (with or without a cmake build) and move its artifacts.""" info(f"Building DEB package {'(no-build): ' if nobuild else ''}{pkg_name}") deb_dir = os.path.join(pkg_dir, "debian") os.makedirs(deb_dir) create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) create_debian_changelog(deb_dir, pkg_name, version, build_number) if nobuild: create_debian_rules_nobuild(deb_dir, skip_shlibdeps=skip_shlibdeps) else: create_debian_rules(deb_dir, cmake_flags, skip_tests) os.makedirs(os.path.join(deb_dir, "source")) with open(os.path.join(deb_dir, "source", "format"), "w") as f: f.write("3.0 (quilt)\n") with open(os.path.join(deb_dir, "compat"), "w") as f: f.write("13\n") if install_lines: create_debian_install(deb_dir, install_lines) info(f"Running debuild for {pkg_name}...") os.chdir(pkg_dir) env = os.environ.copy() env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" run("debuild -us -uc -b", f"debuild failed for {pkg_name}", env=env) parent_dir = os.path.dirname(pkg_dir) for f in os.listdir(parent_dir): if f.endswith((".deb", ".dsc", ".changes", ".buildinfo")): src = os.path.join(parent_dir, f) dst = os.path.join(output_dir, f) if os.path.exists(dst): os.remove(dst) shutil.move(src, output_dir) info(f"Moved {f} to {output_dir}") return True def source_install_lines(pkg_dir, release_dirs): lines = [] for rel_dir in release_dirs: src_dir = os.path.join(pkg_dir, rel_dir) if not os.path.exists(src_dir): continue for root, dirs, files in os.walk(src_dir): dirs[:] = [d for d in dirs if not d.startswith('.')] for file in files: if os.path.splitext(file)[1].lower() in SOURCE_EXTENSIONS or file == "CMakeLists.txt": rel_path = os.path.relpath(os.path.join(root, file), pkg_dir) lines.append(f"{rel_path} {os.path.join('usr/src', os.path.dirname(rel_path))}/") return lines def ctest_runtime_install_lines(pkg_dir, release_dirs): lines = [] for exe_path, module_path in find_test_executables(pkg_dir, release_dirs): lines.append(f"{os.path.relpath(exe_path, pkg_dir)} usr/lib/ctest/") info(f" Including test executable: {os.path.basename(exe_path)} from {module_path}") for ctest_file in ("build4/CTestTestfile.cmake", "build4/DartConfiguration.tcl", "build4/Testing/TAG"): if os.path.exists(os.path.join(pkg_dir, ctest_file)): lines.append(f"{ctest_file} usr/lib/ctest/") return lines def test_sources_install_lines(pkg_dir, release_dirs): return [f"{rel_path} {os.path.dirname(rel_path)}/" for rel_path in find_test_sources(pkg_dir, release_dirs)] def pipeline_install_lines(pkg_dir): lines = [] for pipeline_file in PIPELINE_FILES: if os.path.exists(os.path.join(pkg_dir, pipeline_file)): lines.append(f"{pipeline_file} {os.path.join('usr/src/pipeline', os.path.dirname(pipeline_file))}/") info(f" Added pipeline file: {pipeline_file}") return lines def build_nobuild_package(name, source_dir, build_base, output_dir, version, build_number, description, prepare, allowed_patterns, forbidden_patterns, file_filter=None, skip_shlibdeps=False, section="devel"): """Stage the source tree, build a no-build DEB and validate its contents.""" info(f"\n=== Building {name} ===") pkg_dir = os.path.join(build_base, f"{name}-{version}") os.makedirs(pkg_dir) copy_src_dir_to_dest_dir(source_dir, pkg_dir, file_filter=file_filter) build_deb(pkg_name=name, pkg_dir=pkg_dir, output_dir=output_dir, version=version, build_number=build_number, nobuild=True, skip_shlibdeps=skip_shlibdeps, install_lines=prepare(pkg_dir), description=description, section=section) manifest = create_manifest(output_dir, name, pkg_dir) validate_package_contents(name, manifest, allowed_patterns, forbidden_patterns) def build_all_debs(source_dir, output_dir, version, build_number, cmake_flags): """Build the 5 DEB packages: build once from the source tree, then package.""" info("========================================") info("Building 5 DEB packages for Qt6 Release") info("========================================") os.makedirs(output_dir, exist_ok=True) os.makedirs(os.path.join(output_dir, "manifests"), exist_ok=True) build_base = os.path.join(source_dir, "deb_build_tmp") remove_directory(build_base) os.makedirs(build_base) release_dirs = ["TetraIF", "Z/src", "Lts/Common"] # Step 1: build once from the full source tree info("\n=== Step 1: Building from full source tree ===") build_dir = os.path.join(source_dir, "build") os.makedirs(build_dir, exist_ok=True) os.chdir(build_dir) info("Running CMake configuration...") # Ninja is REQUIRED here: with the CMake 4.x in the legatus-qt6 image the # Unix Makefiles generator emits no rule for AUTOMOC # _autogen/mocs_compilation.cpp ("No rule to make target"), which # breaks every Qt target. Ninja handles automoc correctly. run(f"cmake -G Ninja -DCMAKE_BUILD_TYPE=Release {cmake_flags} ..", "CMake configuration failed") info("Running build...") run(f"ninja -j$(nproc)", "Build failed") info("Build completed successfully") # Package 1: qt6-release-runtime (binaries and libraries, built via debuild) info("\n=== Building qt6-release-runtime ===") pkg1_dir = os.path.join(build_base, f"qt6-release-runtime-{version}") os.makedirs(pkg1_dir) # FULL source tree (needed for debian/rules cmake reference) copy_src_dir_to_dest_dir(source_dir, pkg1_dir) # Install from all component install paths: ALL -> bin/, lib/ ; SV/ ; WS/ install_lines_1 = [ "usr/bin/* usr/bin/", "usr/lib/*.so* usr/lib/", "usr/SV/bin/* usr/bin/", "usr/SV/lib/*.so* usr/lib/", "usr/WS/bin/* usr/bin/", "usr/WS/lib/*.so* usr/lib/", ] build_deb(pkg_name="qt6-release-runtime", pkg_dir=pkg1_dir, output_dir=output_dir, version=version, build_number=build_number, cmake_flags=cmake_flags, skip_tests=True, install_lines=install_lines_1, description="Qt6 Release Runtime - binaries and libraries from release scope S", section="libs") manifest_1 = create_manifest(output_dir, "qt6-release-runtime", pkg1_dir) validate_package_contents("qt6-release-runtime", manifest_1, allowed_patterns=["usr/bin/*", "usr/lib/*", "usr/SV/*", "usr/WS/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["*.cpp", "*.h", "*.hpp", "tests/*", "jenkins/*", "Dockerfile*"]) # Package 2: qt6-release-sources (release-scope sources, DDS excluded) build_nobuild_package( "qt6-release-sources", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release Sources - C++ source files from release scope S", prepare=lambda pkg_dir: source_install_lines(pkg_dir, release_dirs), allowed_patterns=["usr/src/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["build/*", "tests/*", "jenkins/*"], file_filter=is_dds_file) # Package 3: qt6-release-ctest-runtime (test executables from release scope) build_nobuild_package( "qt6-release-ctest-runtime", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release CTest Runtime - executable test programs from release scope", prepare=lambda pkg_dir: ctest_runtime_install_lines(pkg_dir, release_dirs), allowed_patterns=["usr/lib/ctest/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["*.cpp", "*.h", "jenkins/*"], skip_shlibdeps=True) # Package 4: qt6-release-ctest-sources (test source files from release scope) build_nobuild_package( "qt6-release-ctest-sources", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release CTest Sources - test source files from release scope", prepare=lambda pkg_dir: test_sources_install_lines(pkg_dir, release_dirs), allowed_patterns=["Lts/*", "TetraIF/*", "Z/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["build/*", "jenkins/*"]) # Package 5: qt6-release-build-pipeline (Docker, Jenkins, CMake, packaging scripts) build_nobuild_package( "qt6-release-build-pipeline", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release Build Pipeline - Docker, Jenkins, CMake, and packaging scripts", prepare=pipeline_install_lines, allowed_patterns=["usr/src/pipeline/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["*.cpp", "*.h", "tests/*"]) # Cleanup os.chdir(source_dir) remove_directory(build_base) info("\n========================================") info(f"DEB package build completed: {output_dir}") info("Generated packages:") for f in os.listdir(output_dir): if f.endswith(".deb"): info(f" - {f}") info("========================================") def main(): parser = argparse.ArgumentParser(description="Build DEB packages for Legatus Qt6 Release") parser.add_argument("-c", "--config", required=True, help="Config type (LTS)") parser.add_argument("--project_name", required=True, help="Project name (qatar|ahtc|guez)") parser.add_argument("--version", required=True, help="Version (Major.minor)") parser.add_argument("--output_dir", required=True, help="Output directory for packages") parser.add_argument("--source_dir", required=True, help="Source directory path") parser.add_argument("--cmake_build_flags", default="", help="CMake build flags") parser.add_argument("--build_number", default="1", help="Build number") parser.add_argument("--release_scope", action="store_true", help="Only include release scope files") args = parser.parse_args() info(f"Building {args.config} package for {args.project_name} v{args.version}") info(f"Output directory: {args.output_dir}") info(f"Source directory: {args.source_dir}") info(f"CMake flags: {args.cmake_build_flags}") info(f"Release scope: {args.release_scope}") build_all_debs(source_dir=args.source_dir, output_dir=args.output_dir, version=args.version, build_number=args.build_number, cmake_flags=args.cmake_build_flags) if __name__ == "__main__": main()