From df8b739e80afd293bba7f076a236a3b93b6f84f4 Mon Sep 17 00:00:00 2001 From: berrux_dev Date: Fri, 13 Nov 2026 13:26:50 +0100 Subject: [PATCH] Fix rpmbuild filename: add make_rpm_ubuntu26_release2.py (Release 2 DEB producer) new_docker_build_release2.sh and the Release 2 README reference rpmbuild/make_rpm_ubuntu26_release2.py, but only make_rpm_ubuntu26.py existed (used by the jenkins-ci-rme builder). The Release 2 producer is the same general 5-DEB builder (runtime, sources, ctest-runtime, ctest-sources, build-pipeline), so add it under the expected name. make_rpm_ubuntu26.py is kept for the jenkins-ci-rme flow. Co-Authored-By: Claude Code --- .../rpmbuild/make_rpm_ubuntu26_release2.py | 527 ++++++++++++++++++ 1 file changed, 527 insertions(+) create mode 100644 qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26_release2.py diff --git a/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26_release2.py b/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26_release2.py new file mode 100644 index 000000000..562960818 --- /dev/null +++ b/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26_release2.py @@ -0,0 +1,527 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- +""" +DEB package builder for Ubuntu 26.04 - Qt6 Release +Creates 5 separate DEB packages as per customer requirements: +1. qt6-release-runtime - binaries and libraries from S +2. qt6-release-sources - Qt6 C++ sources from S +3. qt6-release-ctest-runtime - CTest executables +4. qt6-release-ctest-sources - CTest source files +5. qt6-release-build-pipeline - Dockerfiles, build scripts, CI/CD files +""" + +import argparse +import fnmatch +import os +import shutil +import subprocess +import sys +from email.utils import formatdate + +SKIP_DIRS = {'build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'} +SOURCE_EXTENSIONS = ('.cpp', '.h', '.hpp', '.cc', '.cxx', '.hxx') +PIPELINE_FILES = [ + "new_docker_build.sh", + "Dockerfile", + "jenkins-ci-rme/Jenkinsfile", + "jenkins-ci-rme/jenkins/Jenkinsfile", + "jenkins-ci-rme/jenkins_builder_ubuntu26.sh", + "jenkins-ci-rme/jenkins_publisher_ubuntu26.sh", + "jenkins-ci-rme/release-16-10-2026.sh", + "rpmbuild/make_rpm_ubuntu26.py", + "CMakeLists.txt", +] + + +def verbose(): + verb = os.getenv("EMERGE_VERBOSE") + if verb is not None and verb.isdigit() and int(verb) > 0: + return int(verb) + return 0 + + +def debug(message): + if verbose() >= 1: + print(f"debug: {message}") + return True + + +def info(message): + print(f"info: {message}") + return True + + +def warning(message): + print(f"warning: {message}") + return True + + +def error(message): + die(message) + + +def die(message): + print(f"emerge fatal error: {message}", file=sys.stderr) + sys.exit(1) + + +def run(cmd, label, env=None): + """Run a shell command, print its output and die on failure.""" + debug(f"Executing: {cmd}") + ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) + if ret.returncode != 0: + print(ret.stdout) + print(ret.stderr) + die(f"{label} with code {ret.returncode}") + + +def remove_directory(path): + if os.path.exists(path): + shutil.rmtree(path) + + +def is_dds_file(filepath): + """True if the path is DDS-related and must be excluded.""" + return 'dds' in filepath.lower() + + +def prune_dirs(dirs, root, file_filter): + """Prune hidden/build dirs in-place (and dirs matching file_filter).""" + dirs[:] = [d for d in dirs + if not d.startswith('.') + and d not in SKIP_DIRS + and (file_filter is None or not file_filter(os.path.join(root, d)))] + + +def copy_src_dir_to_dest_dir(src_dir, dest_dir, file_filter=None): + """Copy the source tree, skipping hidden/build dirs and filtered files.""" + os.makedirs(dest_dir, exist_ok=True) + for root, dirs, files in os.walk(src_dir): + prune_dirs(dirs, root, file_filter) + for file in files: + src_file = os.path.join(root, file) + if file_filter and file_filter(src_file): + continue + dest_file = os.path.join(dest_dir, os.path.relpath(src_file, src_dir)) + os.makedirs(os.path.dirname(dest_file), exist_ok=True) + shutil.copy2(src_file, dest_file) + + +def find_test_module_source(pkg_dir, test_name, release_dirs): + """Module dir (relative to pkg_dir) whose CMakeLists defines test_name, or None.""" + for rel_dir in release_dirs: + module_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(module_dir): + continue + for root, dirs, files in os.walk(module_dir): + prune_dirs(dirs, root, None) + if 'CMakeLists.txt' not in files: + continue + try: + with open(os.path.join(root, 'CMakeLists.txt')) as f: + content = f.read() + except Exception: + continue + if f'ADD_EXECUTABLE({test_name}' in content or f'ADD_TEST(NAME {test_name}' in content: + return os.path.relpath(root, pkg_dir) + return None + + +def find_test_executables(pkg_dir, release_dirs): + """(executable, module) pairs for the test executables in build4/bin/.""" + test_executables = [] + build_bin_dir = os.path.join(pkg_dir, "build4", "bin") + if not os.path.exists(build_bin_dir): + warning(f"Build bin directory not found: {build_bin_dir}") + return test_executables + for filename in os.listdir(build_bin_dir): + file_path = os.path.join(build_bin_dir, filename) + if not (os.path.isfile(file_path) and os.access(file_path, os.X_OK)): + continue + if 'test' not in filename.lower(): + continue + module_path = find_test_module_source(pkg_dir, filename, release_dirs) + if module_path: + test_executables.append((file_path, module_path)) + debug(f" Found test executable: {filename} from {module_path}") + else: + debug(f" Skipping test executable (no module found): {filename}") + return test_executables + + +def find_test_sources(pkg_dir, release_dirs): + """Relative paths of test source files (name containing 'test', .cpp/.h).""" + test_sources = [] + for rel_dir in release_dirs: + module_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(module_dir): + continue + for root, dirs, files in os.walk(module_dir): + prune_dirs(dirs, root, None) + for filename in files: + if 'test' in filename.lower() and filename.endswith(('.cpp', '.h')): + file_path = os.path.join(root, filename) + rel_path = os.path.relpath(file_path, pkg_dir) + test_sources.append(rel_path) + debug(f" Found test source: {rel_path}") + return test_sources + + +def write_deb_file(deb_dir, name, content, executable=False): + """Write a debian/* file and log its creation.""" + path = os.path.join(deb_dir, name) + with open(path, "w") as f: + f.write(content) + if executable: + os.chmod(path, 0o755) + info(f"Created {path}") + + +def create_debian_control(deb_dir, package_name, version, build_number, description, + depends="", section="devel"): + write_deb_file(deb_dir, "control", f"""Source: {package_name} +Section: {section} +Priority: optional +Maintainer: Legatus Team +Build-Depends: debhelper (>= 13), cmake, qt6-base-dev, qt6-tools-dev +Standards-Version: 4.6.0 + +Package: {package_name} +Architecture: any +Depends: ${{shlibs:Depends}}, ${{misc:Depends}}{", " + depends if depends else ""} +Description: {description} + Legatus Qt6 Release Package + This package is part of the Legatus Qt6 migration release. +""") + + +def create_debian_changelog(deb_dir, package_name, version, build_number): + write_deb_file(deb_dir, "changelog", f"""{package_name} ({version}-{build_number}) unstable; urgency=medium + + * Qt6 Release Build + + -- Legatus Team {formatdate(localtime=True)} +""") + + +def create_debian_rules(deb_dir, cmake_flags="", skip_tests=False): + test_override = """ +override_dh_auto_test: + # Tests skipped for this package type +""" if skip_tests else "" + write_deb_file(deb_dir, "rules", f"""#!/usr/bin/make -f +%: + dh $@ --buildsystem=cmake + +override_dh_auto_configure: + dh_auto_configure -- -DCMAKE_BUILD_TYPE=Release {cmake_flags} + +override_dh_auto_build: + dh_auto_build --parallel + +override_dh_auto_install: + dh_auto_install --destdir=debian/tmp +{test_override}""", executable=True) + + +def create_debian_rules_nobuild(deb_dir, skip_shlibdeps=False): + content = """#!/usr/bin/make -f +%: + dh $@ + +override_dh_auto_build: + # No build step needed - pre-built files only + +override_dh_auto_configure: + # No configure step needed - pre-built files only + +override_dh_auto_install: + dh_auto_install +""" + if skip_shlibdeps: + content += """ +override_dh_shlibdeps: + # Skip shlibdeps for test executables - Qt6 deps are external system packages +""" + write_deb_file(deb_dir, "rules", content, executable=True) + + +def create_debian_install(deb_dir, install_lines): + install_file = os.path.join(deb_dir, "install") + with open(install_file, "w") as f: + for line in install_lines: + f.write(line + "\n") + info(f"Created {install_file} with {len(install_lines)} entries") + + +def create_manifest(output_dir, package_name, pkg_dir): + """Write the manifest of package contents (from the DEB, or the pkg dir).""" + manifest_file = os.path.join(output_dir, "manifests", f"{package_name}.manifest") + os.makedirs(os.path.dirname(manifest_file), exist_ok=True) + deb_file = next((os.path.join(output_dir, f) for f in os.listdir(output_dir) + if f.startswith(package_name) and f.endswith(".deb")), None) + manifest_lines = [] + if deb_file and os.path.exists(deb_file): + result = subprocess.run(["dpkg-deb", "--contents", deb_file], + capture_output=True, text=True) + if result.returncode == 0: + # dpkg-deb --contents: "-rw-r--r-- root/root 1234 2024-01-01 00:00 ./path" + for line in result.stdout.strip().split("\n"): + parts = line.split() + if len(parts) >= 6: + file_path = parts[-1] + if file_path.startswith("./"): + file_path = file_path[2:] + elif file_path.startswith("/"): + file_path = file_path[1:] + if file_path: + manifest_lines.append(file_path) + elif os.path.exists(pkg_dir): + info(f"Warning: DEB file not found for {package_name}, scanning directory instead") + for root, dirs, files in os.walk(pkg_dir): + for file in files: + manifest_lines.append(os.path.relpath(os.path.join(root, file), pkg_dir)) + manifest_lines.sort() + with open(manifest_file, "w") as f: + for line in manifest_lines: + f.write(line + "\n") + info(f"Created manifest: {manifest_file} ({len(manifest_lines)} entries)") + return manifest_lines + + +def validate_package_contents(package_name, manifest_lines, allowed_patterns, forbidden_patterns=None): + """Validate package contents against the allowlist/forbidden list.""" + errors = [] + warnings = [] + for file_path in manifest_lines: + if file_path.endswith('/'): + continue + if forbidden_patterns: + for pattern in forbidden_patterns: + if fnmatch.fnmatch(file_path, pattern): + errors.append(f"FORBIDDEN file in {package_name}: {file_path} (matches {pattern})") + allowed = bool(allowed_patterns) and any(fnmatch.fnmatch(file_path, p) for p in allowed_patterns) + if not allowed and allowed_patterns: + warnings.append(f"Unexpected file in {package_name}: {file_path}") + for err in errors: + error(f"Validation error: {err}") + if errors: + return False + for warn in warnings: + warning(f"Validation warning: {warn}") + return True + + +def build_deb(pkg_name, pkg_dir, output_dir, version, build_number, + cmake_flags="", nobuild=False, skip_tests=False, skip_shlibdeps=False, + install_lines=None, description="", section="devel", depends=""): + """Build one DEB package (with or without a cmake build) and move its artifacts.""" + info(f"Building DEB package {'(no-build): ' if nobuild else ''}{pkg_name}") + deb_dir = os.path.join(pkg_dir, "debian") + os.makedirs(deb_dir) + create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) + create_debian_changelog(deb_dir, pkg_name, version, build_number) + if nobuild: + create_debian_rules_nobuild(deb_dir, skip_shlibdeps=skip_shlibdeps) + else: + create_debian_rules(deb_dir, cmake_flags, skip_tests) + os.makedirs(os.path.join(deb_dir, "source")) + with open(os.path.join(deb_dir, "source", "format"), "w") as f: + f.write("3.0 (quilt)\n") + with open(os.path.join(deb_dir, "compat"), "w") as f: + f.write("13\n") + if install_lines: + create_debian_install(deb_dir, install_lines) + + info(f"Running debuild for {pkg_name}...") + os.chdir(pkg_dir) + env = os.environ.copy() + env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" + run("debuild -us -uc -b", f"debuild failed for {pkg_name}", env=env) + + parent_dir = os.path.dirname(pkg_dir) + for f in os.listdir(parent_dir): + if f.endswith((".deb", ".dsc", ".changes", ".buildinfo")): + src = os.path.join(parent_dir, f) + dst = os.path.join(output_dir, f) + if os.path.exists(dst): + os.remove(dst) + shutil.move(src, output_dir) + info(f"Moved {f} to {output_dir}") + return True + + +def source_install_lines(pkg_dir, release_dirs): + lines = [] + for rel_dir in release_dirs: + src_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(src_dir): + continue + for root, dirs, files in os.walk(src_dir): + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if os.path.splitext(file)[1].lower() in SOURCE_EXTENSIONS or file == "CMakeLists.txt": + rel_path = os.path.relpath(os.path.join(root, file), pkg_dir) + lines.append(f"{rel_path} {os.path.join('usr/src', os.path.dirname(rel_path))}/") + return lines + + +def ctest_runtime_install_lines(pkg_dir, release_dirs): + lines = [] + for exe_path, module_path in find_test_executables(pkg_dir, release_dirs): + lines.append(f"{os.path.relpath(exe_path, pkg_dir)} usr/lib/ctest/") + info(f" Including test executable: {os.path.basename(exe_path)} from {module_path}") + for ctest_file in ("build4/CTestTestfile.cmake", "build4/DartConfiguration.tcl", "build4/Testing/TAG"): + if os.path.exists(os.path.join(pkg_dir, ctest_file)): + lines.append(f"{ctest_file} usr/lib/ctest/") + return lines + + +def test_sources_install_lines(pkg_dir, release_dirs): + return [f"{rel_path} {os.path.dirname(rel_path)}/" for rel_path in find_test_sources(pkg_dir, release_dirs)] + + +def pipeline_install_lines(pkg_dir): + lines = [] + for pipeline_file in PIPELINE_FILES: + if os.path.exists(os.path.join(pkg_dir, pipeline_file)): + lines.append(f"{pipeline_file} {os.path.join('usr/src/pipeline', os.path.dirname(pipeline_file))}/") + info(f" Added pipeline file: {pipeline_file}") + return lines + + +def build_nobuild_package(name, source_dir, build_base, output_dir, version, build_number, + description, prepare, allowed_patterns, forbidden_patterns, + file_filter=None, skip_shlibdeps=False, section="devel"): + """Stage the source tree, build a no-build DEB and validate its contents.""" + info(f"\n=== Building {name} ===") + pkg_dir = os.path.join(build_base, f"{name}-{version}") + os.makedirs(pkg_dir) + copy_src_dir_to_dest_dir(source_dir, pkg_dir, file_filter=file_filter) + build_deb(pkg_name=name, pkg_dir=pkg_dir, output_dir=output_dir, + version=version, build_number=build_number, nobuild=True, + skip_shlibdeps=skip_shlibdeps, install_lines=prepare(pkg_dir), + description=description, section=section) + manifest = create_manifest(output_dir, name, pkg_dir) + validate_package_contents(name, manifest, allowed_patterns, forbidden_patterns) + + +def build_all_debs(source_dir, output_dir, version, build_number, cmake_flags): + """Build the 5 DEB packages: build once from the source tree, then package.""" + info("========================================") + info("Building 5 DEB packages for Qt6 Release") + info("========================================") + os.makedirs(output_dir, exist_ok=True) + os.makedirs(os.path.join(output_dir, "manifests"), exist_ok=True) + build_base = os.path.join(source_dir, "deb_build_tmp") + remove_directory(build_base) + os.makedirs(build_base) + release_dirs = ["TetraIF", "Z/src", "Lts/Common"] + + # Step 1: build once from the full source tree + info("\n=== Step 1: Building from full source tree ===") + build_dir = os.path.join(source_dir, "build") + os.makedirs(build_dir, exist_ok=True) + os.chdir(build_dir) + info("Running CMake configuration...") + run(f"cmake -DCMAKE_BUILD_TYPE=Release {cmake_flags} ..", "CMake configuration failed") + info("Running build...") + run(f"make -j$(nproc)", "Build failed") + info("Build completed successfully") + + # Package 1: qt6-release-runtime (binaries and libraries, built via debuild) + info("\n=== Building qt6-release-runtime ===") + pkg1_dir = os.path.join(build_base, f"qt6-release-runtime-{version}") + os.makedirs(pkg1_dir) + # FULL source tree (needed for debian/rules cmake reference) + copy_src_dir_to_dest_dir(source_dir, pkg1_dir) + # Install from all component install paths: ALL -> bin/, lib/ ; SV/ ; WS/ + install_lines_1 = [ + "usr/bin/* usr/bin/", + "usr/lib/*.so* usr/lib/", + "usr/SV/bin/* usr/bin/", + "usr/SV/lib/*.so* usr/lib/", + "usr/WS/bin/* usr/bin/", + "usr/WS/lib/*.so* usr/lib/", + ] + build_deb(pkg_name="qt6-release-runtime", pkg_dir=pkg1_dir, output_dir=output_dir, + version=version, build_number=build_number, cmake_flags=cmake_flags, + skip_tests=True, install_lines=install_lines_1, + description="Qt6 Release Runtime - binaries and libraries from release scope S", + section="libs") + manifest_1 = create_manifest(output_dir, "qt6-release-runtime", pkg1_dir) + validate_package_contents("qt6-release-runtime", manifest_1, + allowed_patterns=["usr/bin/*", "usr/lib/*", "usr/SV/*", "usr/WS/*", "usr/share/doc/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "*.hpp", "tests/*", "jenkins/*", "Dockerfile*"]) + + # Package 2: qt6-release-sources (release-scope sources, DDS excluded) + build_nobuild_package( + "qt6-release-sources", source_dir, build_base, output_dir, version, build_number, + description="Qt6 Release Sources - C++ source files from release scope S", + prepare=lambda pkg_dir: source_install_lines(pkg_dir, release_dirs), + allowed_patterns=["usr/src/*", "usr/share/doc/*", "debian/*"], + forbidden_patterns=["build/*", "tests/*", "jenkins/*"], + file_filter=is_dds_file) + + # Package 3: qt6-release-ctest-runtime (test executables from release scope) + build_nobuild_package( + "qt6-release-ctest-runtime", source_dir, build_base, output_dir, version, build_number, + description="Qt6 Release CTest Runtime - executable test programs from release scope", + prepare=lambda pkg_dir: ctest_runtime_install_lines(pkg_dir, release_dirs), + allowed_patterns=["usr/lib/ctest/*", "usr/share/doc/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "jenkins/*"], + skip_shlibdeps=True) + + # Package 4: qt6-release-ctest-sources (test source files from release scope) + build_nobuild_package( + "qt6-release-ctest-sources", source_dir, build_base, output_dir, version, build_number, + description="Qt6 Release CTest Sources - test source files from release scope", + prepare=lambda pkg_dir: test_sources_install_lines(pkg_dir, release_dirs), + allowed_patterns=["Lts/*", "TetraIF/*", "Z/*", "usr/share/doc/*", "debian/*"], + forbidden_patterns=["build/*", "jenkins/*"]) + + # Package 5: qt6-release-build-pipeline (Docker, Jenkins, CMake, packaging scripts) + build_nobuild_package( + "qt6-release-build-pipeline", source_dir, build_base, output_dir, version, build_number, + description="Qt6 Release Build Pipeline - Docker, Jenkins, CMake, and packaging scripts", + prepare=pipeline_install_lines, + allowed_patterns=["usr/src/pipeline/*", "usr/share/doc/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "tests/*"]) + + # Cleanup + os.chdir(source_dir) + remove_directory(build_base) + + info("\n========================================") + info(f"DEB package build completed: {output_dir}") + info("Generated packages:") + for f in os.listdir(output_dir): + if f.endswith(".deb"): + info(f" - {f}") + info("========================================") + + +def main(): + parser = argparse.ArgumentParser(description="Build DEB packages for Legatus Qt6 Release") + parser.add_argument("-c", "--config", required=True, help="Config type (LTS)") + parser.add_argument("--project_name", required=True, help="Project name (qatar|ahtc|guez)") + parser.add_argument("--version", required=True, help="Version (Major.minor)") + parser.add_argument("--output_dir", required=True, help="Output directory for packages") + parser.add_argument("--source_dir", required=True, help="Source directory path") + parser.add_argument("--cmake_build_flags", default="", help="CMake build flags") + parser.add_argument("--build_number", default="1", help="Build number") + parser.add_argument("--release_scope", action="store_true", help="Only include release scope files") + + args = parser.parse_args() + info(f"Building {args.config} package for {args.project_name} v{args.version}") + info(f"Output directory: {args.output_dir}") + info(f"Source directory: {args.source_dir}") + info(f"CMake flags: {args.cmake_build_flags}") + info(f"Release scope: {args.release_scope}") + build_all_debs(source_dir=args.source_dir, output_dir=args.output_dir, + version=args.version, build_number=args.build_number, + cmake_flags=args.cmake_build_flags) + + +if __name__ == "__main__": + main()