diff --git a/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26.py b/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26.py index 12615ee13..ca3973a06 100755 --- a/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26.py +++ b/qt6-after/qt6-migrated-legatus/rpmbuild/make_rpm_ubuntu26.py @@ -11,16 +11,25 @@ Creates 5 separate DEB packages as per customer requirements: """ import argparse -import configparser -import getpass +import fnmatch import os -import re import shutil -import stat import subprocess import sys -from datetime import datetime -from pathlib import Path +from email.utils import formatdate + +SKIP_DIRS = {'build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'} +SOURCE_EXTENSIONS = ('.cpp', '.h', '.hpp', '.cc', '.cxx', '.hxx') +PIPELINE_FILES = [ + "new_docker_build.sh", + "Dockerfile", + "jenkins-ci-rme/Jenkinsfile", + "jenkins-ci-rme/jenkins_builder_ubuntu26.sh", + "jenkins-ci-rme/jenkins_publisher_ubuntu26.sh", + "jenkins-ci-rme/release-16-10-2026.sh", + "rpmbuild/make_rpm_ubuntu26.py", + "CMakeLists.txt", +] def verbose(): @@ -37,8 +46,7 @@ def debug(message): def info(message): - if verbose() >= 0: - print(f"info: {message}") + print(f"info: {message}") return True @@ -56,227 +64,121 @@ def die(message): sys.exit(1) +def run(cmd, label, env=None): + """Run a shell command, print its output and die on failure.""" + debug(f"Executing: {cmd}") + ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) + if ret.returncode != 0: + print(ret.stdout) + print(ret.stderr) + die(f"{label} with code {ret.returncode}") + + def remove_directory(path): - """Remove directory tree""" if os.path.exists(path): shutil.rmtree(path) -def copy_file_to_dest_dir(src_file, dest_dir): - """Copy a single file to destination directory""" - os.makedirs(dest_dir, exist_ok=True) - shutil.copy2(src_file, dest_dir) - - def is_dds_file(filepath): - """Check if a file or directory path contains DDS-related content that should be excluded - - Args: - filepath: Path to check - - Returns: - True if file should be excluded (is DDS-related), False otherwise - """ - # Convert to lowercase for case-insensitive matching - path_lower = filepath.lower() - - # Exclude files/directories with 'dds' in the name - if 'dds' in path_lower: - return True - - return False + """True if the path is DDS-related and must be excluded.""" + return 'dds' in filepath.lower() -def copy_src_dir_to_dest_dir(src_dir, dest_dir, release_scope=False, file_filter=None): - """Copy source directory to destination with optional filtering - - Args: - src_dir: Source directory - dest_dir: Destination directory - release_scope: If True, only copy release scope directories - file_filter: Optional function to filter files (filepath) -> bool - """ +def prune_dirs(dirs, root, file_filter): + """Prune hidden/build dirs in-place (and dirs matching file_filter).""" + dirs[:] = [d for d in dirs + if not d.startswith('.') + and d not in SKIP_DIRS + and (file_filter is None or not file_filter(os.path.join(root, d)))] + + +def copy_src_dir_to_dest_dir(src_dir, dest_dir, file_filter=None): + """Copy the source tree, skipping hidden/build dirs and filtered files.""" os.makedirs(dest_dir, exist_ok=True) - - # Release scope directories - release_dirs = [ - "TetraIF", - "Z/src", - "Lts/Common" - ] - - if release_scope: - info(f"Copying only release scope directories: {release_dirs}") - # Also copy root-level build files needed for CMake - root_build_files = ['CMakeLists.txt', 'CTestCustom.ctest'] - root_build_dirs = ['cmake'] - - for bf in root_build_files: - src_file = os.path.join(src_dir, bf) - if os.path.exists(src_file): - debug(f" Copying root file {bf}...") - shutil.copy2(src_file, os.path.join(dest_dir, bf)) - - for bd in root_build_dirs: - src_bdir = os.path.join(src_dir, bd) - dest_bdir = os.path.join(dest_dir, bd) - if os.path.exists(src_bdir): - debug(f" Copying root directory {bd}...") - shutil.copytree(src_bdir, dest_bdir, dirs_exist_ok=True) - - for rel_dir in release_dirs: - src_subdir = os.path.join(src_dir, rel_dir) - dest_subdir = os.path.join(dest_dir, rel_dir) - if os.path.exists(src_subdir): - debug(f" Copying {rel_dir}...") - for root, dirs, files in os.walk(src_subdir): - # Skip build directories, hidden directories, deb_build_tmp (prevent recursion), and DDS-related directories - dirs[:] = [d for d in dirs if not d.startswith('.') - and d not in ['build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'] - and (file_filter is None or not file_filter(os.path.join(root, d)))] - - for file in files: - src_file = os.path.join(root, file) - rel_path = os.path.relpath(src_file, src_subdir) - dest_file = os.path.join(dest_subdir, rel_path) - - # Apply file filter if provided (skip file if filter returns True) - if file_filter and file_filter(src_file): - continue - - os.makedirs(os.path.dirname(dest_file), exist_ok=True) - shutil.copy2(src_file, dest_file) - else: - # Copy entire source tree - for root, dirs, files in os.walk(src_dir): - # Skip build directories, hidden directories, deb_build_tmp (prevent recursion), and DDS-related directories - dirs[:] = [d for d in dirs if not d.startswith('.') - and d not in ['build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'] - and (file_filter is None or not file_filter(os.path.join(root, d)))] - - for file in files: - src_file = os.path.join(root, file) - rel_path = os.path.relpath(src_file, src_dir) - dest_file = os.path.join(dest_dir, rel_path) - - # Apply file filter if provided (skip file if filter returns True) - if file_filter and file_filter(src_file): - continue - - os.makedirs(os.path.dirname(dest_file), exist_ok=True) - shutil.copy2(src_file, dest_file) + for root, dirs, files in os.walk(src_dir): + prune_dirs(dirs, root, file_filter) + for file in files: + src_file = os.path.join(root, file) + if file_filter and file_filter(src_file): + continue + dest_file = os.path.join(dest_dir, os.path.relpath(src_file, src_dir)) + os.makedirs(os.path.dirname(dest_file), exist_ok=True) + shutil.copy2(src_file, dest_file) + + +def find_test_module_source(pkg_dir, test_name, release_dirs): + """Module dir (relative to pkg_dir) whose CMakeLists defines test_name, or None.""" + for rel_dir in release_dirs: + module_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(module_dir): + continue + for root, dirs, files in os.walk(module_dir): + prune_dirs(dirs, root, None) + if 'CMakeLists.txt' not in files: + continue + try: + with open(os.path.join(root, 'CMakeLists.txt')) as f: + content = f.read() + except Exception: + continue + if f'ADD_EXECUTABLE({test_name}' in content or f'ADD_TEST(NAME {test_name}' in content: + return os.path.relpath(root, pkg_dir) + return None def find_test_executables(pkg_dir, release_dirs): - """Find test executables from release scope modules in build4/bin/ - - Args: - pkg_dir: Package staging directory - release_dirs: List of release scope directory paths - - Returns: - List of (executable_path, module_path) tuples for tests in release scope - """ + """(executable, module) pairs for the test executables in build4/bin/.""" test_executables = [] build_bin_dir = os.path.join(pkg_dir, "build4", "bin") - if not os.path.exists(build_bin_dir): warning(f"Build bin directory not found: {build_bin_dir}") return test_executables - - # Get all executable files in build4/bin/ for filename in os.listdir(build_bin_dir): file_path = os.path.join(build_bin_dir, filename) - if not os.path.isfile(file_path) or not os.access(file_path, os.X_OK): + if not (os.path.isfile(file_path) and os.access(file_path, os.X_OK)): continue - - # Skip non-test executables (common patterns) - if not ('test' in filename.lower() or 'Test' in filename): + if 'test' not in filename.lower(): continue - - # Find which module this test belongs to by searching CMakeLists.txt module_path = find_test_module_source(pkg_dir, filename, release_dirs) if module_path: test_executables.append((file_path, module_path)) debug(f" Found test executable: {filename} from {module_path}") else: debug(f" Skipping test executable (no module found): {filename}") - return test_executables -def find_test_module_source(pkg_dir, test_name, release_dirs): - """Find the source module directory for a test executable - - Args: - pkg_dir: Package staging directory - test_name: Name of the test executable - release_dirs: List of release scope directory paths - - Returns: - Relative path to module directory, or None if not found - """ - # Search for CMakeLists.txt files that define this test - for rel_dir in release_dirs: - module_dir = os.path.join(pkg_dir, rel_dir) - if not os.path.exists(module_dir): - continue - - for root, dirs, files in os.walk(module_dir): - # Skip build directories - dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['build', 'cmake-build-debug', 'cmake-build-release']] - - if 'CMakeLists.txt' in files: - cmake_file = os.path.join(root, 'CMakeLists.txt') - try: - with open(cmake_file, 'r') as f: - content = f.read() - # Look for ADD_EXECUTABLE or ADD_TEST with this test name - if f'ADD_EXECUTABLE({test_name}' in content or f'ADD_TEST(NAME {test_name}' in content: - # Return the module directory relative to pkg_dir - return os.path.relpath(root, pkg_dir) - except Exception: - pass - - return None - - def find_test_sources(pkg_dir, release_dirs): - """Find test source files in release scope modules - - Args: - pkg_dir: Package staging directory - release_dirs: List of release scope directory paths - - Returns: - List of relative paths to test source files - """ + """Relative paths of test source files (name containing 'test', .cpp/.h).""" test_sources = [] - test_patterns = ['*test*.cpp', '*test*.h', '*Test*.cpp', '*Test*.h', 'test_*.cpp', 'test_*.h'] - for rel_dir in release_dirs: module_dir = os.path.join(pkg_dir, rel_dir) if not os.path.exists(module_dir): continue - for root, dirs, files in os.walk(module_dir): - # Skip build directories and hidden directories - dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['build', 'cmake-build-debug', 'cmake-build-release']] - + prune_dirs(dirs, root, None) for filename in files: - # Check if this is a test file - if 'test' in filename.lower() and (filename.endswith('.cpp') or filename.endswith('.h')): + if 'test' in filename.lower() and filename.endswith(('.cpp', '.h')): file_path = os.path.join(root, filename) rel_path = os.path.relpath(file_path, pkg_dir) test_sources.append(rel_path) debug(f" Found test source: {rel_path}") - return test_sources -def create_debian_control(deb_dir, package_name, version, build_number, description, depends="", section="devel"): - """Create debian/control file for a package""" - control_content = f"""Source: {package_name} +def write_deb_file(deb_dir, name, content, executable=False): + """Write a debian/* file and log its creation.""" + path = os.path.join(deb_dir, name) + with open(path, "w") as f: + f.write(content) + if executable: + os.chmod(path, 0o755) + info(f"Created {path}") + + +def create_debian_control(deb_dir, package_name, version, build_number, description, + depends="", section="devel"): + write_deb_file(deb_dir, "control", f"""Source: {package_name} Section: {section} Priority: optional Maintainer: Legatus Team @@ -289,42 +191,24 @@ Depends: ${{shlibs:Depends}}, ${{misc:Depends}}{", " + depends if depends else " Description: {description} Legatus Qt6 Release Package This package is part of the Legatus Qt6 migration release. -""" - - control_file = os.path.join(deb_dir, "control") - with open(control_file, "w") as f: - f.write(control_content) - info(f"Created {control_file}") +""") def create_debian_changelog(deb_dir, package_name, version, build_number): - """Create debian/changelog file""" - # Use +0000 as timezone offset (UTC) for reproducibility - from email.utils import formatdate - timestamp = formatdate(localtime=True) - changelog_content = f"""{package_name} ({version}-{build_number}) unstable; urgency=medium + write_deb_file(deb_dir, "changelog", f"""{package_name} ({version}-{build_number}) unstable; urgency=medium * Qt6 Release Build - -- Legatus Team {timestamp} -""" - - changelog_file = os.path.join(deb_dir, "changelog") - with open(changelog_file, "w") as f: - f.write(changelog_content) - info(f"Created {changelog_file}") + -- Legatus Team {formatdate(localtime=True)} +""") def create_debian_rules(deb_dir, cmake_flags="", skip_tests=False): - """Create debian/rules file""" - test_override = "" - if skip_tests: - test_override = """ + test_override = """ override_dh_auto_test: # Tests skipped for this package type -""" - - rules_content = f"""#!/usr/bin/make -f +""" if skip_tests else "" + write_deb_file(deb_dir, "rules", f"""#!/usr/bin/make -f %: dh $@ --buildsystem=cmake @@ -336,23 +220,11 @@ override_dh_auto_build: override_dh_auto_install: dh_auto_install --destdir=debian/tmp -{test_override}""" - - rules_file = os.path.join(deb_dir, "rules") - with open(rules_file, "w") as f: - f.write(rules_content) - os.chmod(rules_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH) - info(f"Created {rules_file}") +{test_override}""", executable=True) def create_debian_rules_nobuild(deb_dir, skip_shlibdeps=False): - """Create debian/rules file for packages that don't need building (pre-built files only) - - Args: - deb_dir: Path to debian directory - skip_shlibdeps: If True, skip dh_shlibdeps (useful for test executables with external deps) - """ - rules_content = """#!/usr/bin/make -f + content = """#!/usr/bin/make -f %: dh $@ @@ -366,26 +238,14 @@ override_dh_auto_install: dh_auto_install """ if skip_shlibdeps: - rules_content += """ + content += """ override_dh_shlibdeps: # Skip shlibdeps for test executables - Qt6 deps are external system packages """ - - rules_file = os.path.join(deb_dir, "rules") - with open(rules_file, "w") as f: - f.write(rules_content) - os.chmod(rules_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH) - info(f"Created {rules_file}") - - rules_file = os.path.join(deb_dir, "rules") - with open(rules_file, "w") as f: - f.write(rules_content) - os.chmod(rules_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH) - info(f"Created {rules_file}") + write_deb_file(deb_dir, "rules", content, executable=True) def create_debian_install(deb_dir, install_lines): - """Create debian/install file from list of file entries""" install_file = os.path.join(deb_dir, "install") with open(install_file, "w") as f: for line in install_lines: @@ -394,489 +254,243 @@ def create_debian_install(deb_dir, install_lines): def create_manifest(output_dir, package_name, pkg_dir): - """Create manifest file listing all package contents from actual DEB file""" + """Write the manifest of package contents (from the DEB, or the pkg dir).""" manifest_file = os.path.join(output_dir, "manifests", f"{package_name}.manifest") os.makedirs(os.path.dirname(manifest_file), exist_ok=True) - - # Find the DEB file for this package - deb_file = None - for f in os.listdir(output_dir): - if f.startswith(package_name) and f.endswith(".deb"): - deb_file = os.path.join(output_dir, f) - break - + deb_file = next((os.path.join(output_dir, f) for f in os.listdir(output_dir) + if f.startswith(package_name) and f.endswith(".deb")), None) manifest_lines = [] if deb_file and os.path.exists(deb_file): - # Extract DEB contents using dpkg-deb - result = subprocess.run( - ["dpkg-deb", "--contents", deb_file], - capture_output=True, text=True - ) + result = subprocess.run(["dpkg-deb", "--contents", deb_file], + capture_output=True, text=True) if result.returncode == 0: + # dpkg-deb --contents: "-rw-r--r-- root/root 1234 2024-01-01 00:00 ./path" for line in result.stdout.strip().split("\n"): - if line.strip(): - # dpkg-deb --contents output format: -rw-r--r-- root/root 1234 2024-01-01 00:00 ./path/to/file - # Extract the path (last field) - parts = line.split() - if len(parts) >= 6: - file_path = parts[-1] - # Remove leading ./ or / - if file_path.startswith("./"): - file_path = file_path[2:] - elif file_path.startswith("/"): - file_path = file_path[1:] - if file_path: - manifest_lines.append(file_path) + parts = line.split() + if len(parts) >= 6: + file_path = parts[-1] + if file_path.startswith("./"): + file_path = file_path[2:] + elif file_path.startswith("/"): + file_path = file_path[1:] + if file_path: + manifest_lines.append(file_path) elif os.path.exists(pkg_dir): - # Fallback: scan directory if DEB not found info(f"Warning: DEB file not found for {package_name}, scanning directory instead") for root, dirs, files in os.walk(pkg_dir): for file in files: - file_path = os.path.join(root, file) - rel_path = os.path.relpath(file_path, pkg_dir) - manifest_lines.append(rel_path) - + manifest_lines.append(os.path.relpath(os.path.join(root, file), pkg_dir)) manifest_lines.sort() with open(manifest_file, "w") as f: for line in manifest_lines: f.write(line + "\n") - info(f"Created manifest: {manifest_file} ({len(manifest_lines)} entries)") return manifest_lines def validate_package_contents(package_name, manifest_lines, allowed_patterns, forbidden_patterns=None): - """Validate package contents against allowlist/forbidden list""" - import fnmatch - + """Validate package contents against the allowlist/forbidden list.""" errors = [] warnings = [] - for file_path in manifest_lines: - # Skip directory entries (trailing '/') - only files are allowlisted if file_path.endswith('/'): continue - - # Check forbidden patterns if forbidden_patterns: for pattern in forbidden_patterns: if fnmatch.fnmatch(file_path, pattern): errors.append(f"FORBIDDEN file in {package_name}: {file_path} (matches {pattern})") - - # Check if file matches any allowed pattern - allowed = False - if allowed_patterns: - for pattern in allowed_patterns: - if fnmatch.fnmatch(file_path, pattern): - allowed = True - break - + allowed = bool(allowed_patterns) and any(fnmatch.fnmatch(file_path, p) for p in allowed_patterns) if not allowed and allowed_patterns: warnings.append(f"Unexpected file in {package_name}: {file_path}") - + for err in errors: + error(f"Validation error: {err}") if errors: - for err in errors: - error(f"Validation error: {err}") return False - - if warnings: - for warn in warnings: - warning(f"Validation warning: {warn}") - + for warn in warnings: + warning(f"Validation warning: {warn}") return True -def build_single_deb(pkg_name, pkg_dir, output_dir, version, build_number, cmake_flags, skip_tests, install_lines, description, section="devel", depends=""): - """Build a single DEB package""" - info(f"Building DEB package: {pkg_name}") - +def build_deb(pkg_name, pkg_dir, output_dir, version, build_number, + cmake_flags="", nobuild=False, skip_tests=False, skip_shlibdeps=False, + install_lines=None, description="", section="devel", depends=""): + """Build one DEB package (with or without a cmake build) and move its artifacts.""" + info(f"Building DEB package {'(no-build): ' if nobuild else ''}{pkg_name}") deb_dir = os.path.join(pkg_dir, "debian") os.makedirs(deb_dir) - create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) create_debian_changelog(deb_dir, pkg_name, version, build_number) - create_debian_rules(deb_dir, cmake_flags, skip_tests) - - # Create debian/source/format - source_dir_path = os.path.join(deb_dir, "source") - os.makedirs(source_dir_path) - with open(os.path.join(source_dir_path, "format"), "w") as f: + if nobuild: + create_debian_rules_nobuild(deb_dir, skip_shlibdeps=skip_shlibdeps) + else: + create_debian_rules(deb_dir, cmake_flags, skip_tests) + os.makedirs(os.path.join(deb_dir, "source")) + with open(os.path.join(deb_dir, "source", "format"), "w") as f: f.write("3.0 (quilt)\n") - - # Create debian/compat with open(os.path.join(deb_dir, "compat"), "w") as f: f.write("13\n") - - # Create debian/install if install_lines: create_debian_install(deb_dir, install_lines) - - # Build the package + info(f"Running debuild for {pkg_name}...") os.chdir(pkg_dir) - env = os.environ.copy() env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" - - cmd = f"debuild -us -uc -b" - debug(f"Executing: {cmd}") - ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) - - if ret.returncode != 0: - print(ret.stdout) - print(ret.stderr) - die(f"debuild failed for {pkg_name} with code {ret.returncode}") - - # Move generated packages to output directory + run("debuild -us -uc -b", f"debuild failed for {pkg_name}", env=env) + parent_dir = os.path.dirname(pkg_dir) for f in os.listdir(parent_dir): - if f.endswith(".deb") or f.endswith(".dsc") or f.endswith(".changes") or f.endswith(".buildinfo"): + if f.endswith((".deb", ".dsc", ".changes", ".buildinfo")): src = os.path.join(parent_dir, f) dst = os.path.join(output_dir, f) if os.path.exists(dst): os.remove(dst) shutil.move(src, output_dir) info(f"Moved {f} to {output_dir}") - return True -def build_single_deb_nobuild(pkg_name, pkg_dir, output_dir, version, build_number, install_lines, description, section="devel", depends="", skip_shlibdeps=False): - """Build a single DEB package without running cmake/make (pre-built files only) - - Args: - skip_shlibdeps: If True, skip dh_shlibdeps (useful for test executables with external deps) - """ - info(f"Building DEB package (no-build): {pkg_name}") - - deb_dir = os.path.join(pkg_dir, "debian") - os.makedirs(deb_dir) - - create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) - create_debian_changelog(deb_dir, pkg_name, version, build_number) - create_debian_rules_nobuild(deb_dir, skip_shlibdeps=skip_shlibdeps) - - # Create debian/source/format - source_dir_path = os.path.join(deb_dir, "source") - os.makedirs(source_dir_path) - with open(os.path.join(source_dir_path, "format"), "w") as f: - f.write("3.0 (quilt)\n") - - # Create debian/compat - with open(os.path.join(deb_dir, "compat"), "w") as f: - f.write("13\n") - - # Create debian/install - if install_lines: - create_debian_install(deb_dir, install_lines) - - # Build the package - info(f"Running debuild for {pkg_name}...") - os.chdir(pkg_dir) - - env = os.environ.copy() - env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" - - cmd = f"debuild -us -uc -b" - debug(f"Executing: {cmd}") - ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) - - if ret.returncode != 0: - print(ret.stdout) - print(ret.stderr) - die(f"debuild failed for {pkg_name} with code {ret.returncode}") - - # Move generated packages to output directory - parent_dir = os.path.dirname(pkg_dir) - for f in os.listdir(parent_dir): - if f.endswith(".deb") or f.endswith(".dsc") or f.endswith(".changes") or f.endswith(".buildinfo"): - src = os.path.join(parent_dir, f) - dst = os.path.join(output_dir, f) - if os.path.exists(dst): - os.remove(dst) - shutil.move(src, output_dir) - info(f"Moved {f} to {output_dir}") - - return True +def source_install_lines(pkg_dir, release_dirs): + lines = [] + for rel_dir in release_dirs: + src_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(src_dir): + continue + for root, dirs, files in os.walk(src_dir): + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + if os.path.splitext(file)[1].lower() in SOURCE_EXTENSIONS or file == "CMakeLists.txt": + rel_path = os.path.relpath(os.path.join(root, file), pkg_dir) + lines.append(f"{rel_path} {os.path.join('usr/src', os.path.dirname(rel_path))}/") + return lines -def build_all_debs(source_dir, output_dir, project_name, version, build_number, cmake_flags, release_scope=True): - """Build all 5 required DEB packages - build once, then package""" +def ctest_runtime_install_lines(pkg_dir, release_dirs): + lines = [] + for exe_path, module_path in find_test_executables(pkg_dir, release_dirs): + lines.append(f"{os.path.relpath(exe_path, pkg_dir)} usr/lib/ctest/") + info(f" Including test executable: {os.path.basename(exe_path)} from {module_path}") + for ctest_file in ("build4/CTestTestfile.cmake", "build4/DartConfiguration.tcl", "build4/Testing/TAG"): + if os.path.exists(os.path.join(pkg_dir, ctest_file)): + lines.append(f"{ctest_file} usr/lib/ctest/") + return lines + + +def test_sources_install_lines(pkg_dir, release_dirs): + return [f"{rel_path} {os.path.dirname(rel_path)}/" for rel_path in find_test_sources(pkg_dir, release_dirs)] + + +def pipeline_install_lines(pkg_dir): + lines = [] + for pipeline_file in PIPELINE_FILES: + if os.path.exists(os.path.join(pkg_dir, pipeline_file)): + lines.append(f"{pipeline_file} {os.path.join('usr/src/pipeline', os.path.dirname(pipeline_file))}/") + info(f" Added pipeline file: {pipeline_file}") + return lines + + +def build_nobuild_package(name, source_dir, build_base, output_dir, version, build_number, + description, prepare, allowed_patterns, forbidden_patterns, + file_filter=None, skip_shlibdeps=False, section="devel"): + """Stage the source tree, build a no-build DEB and validate its contents.""" + info(f"\n=== Building {name} ===") + pkg_dir = os.path.join(build_base, f"{name}-{version}") + os.makedirs(pkg_dir) + copy_src_dir_to_dest_dir(source_dir, pkg_dir, file_filter=file_filter) + build_deb(pkg_name=name, pkg_dir=pkg_dir, output_dir=output_dir, + version=version, build_number=build_number, nobuild=True, + skip_shlibdeps=skip_shlibdeps, install_lines=prepare(pkg_dir), + description=description, section=section) + manifest = create_manifest(output_dir, name, pkg_dir) + validate_package_contents(name, manifest, allowed_patterns, forbidden_patterns) + + +def build_all_debs(source_dir, output_dir, version, build_number, cmake_flags): + """Build the 5 DEB packages: build once from the source tree, then package.""" info("========================================") info("Building 5 DEB packages for Qt6 Release") info("========================================") - os.makedirs(output_dir, exist_ok=True) os.makedirs(os.path.join(output_dir, "manifests"), exist_ok=True) - build_base = os.path.join(source_dir, "deb_build_tmp") remove_directory(build_base) os.makedirs(build_base) - - # Release scope directories release_dirs = ["TetraIF", "Z/src", "Lts/Common"] - - # Source file extensions - source_extensions = ['.cpp', '.h', '.hpp', '.cc', '.cxx', '.hxx'] - script_extensions = ['.py', '.sh', '.cmake', '.txt'] - - # ============================================ - # STEP 1: Build once from full source tree - # ============================================ + + # Step 1: build once from the full source tree info("\n=== Step 1: Building from full source tree ===") build_dir = os.path.join(source_dir, "build") os.makedirs(build_dir, exist_ok=True) os.chdir(build_dir) - - # Configure info("Running CMake configuration...") - cmake_cmd = f"cmake -DCMAKE_BUILD_TYPE=Release {cmake_flags} .." - ret = subprocess.run(cmake_cmd, shell=True, capture_output=True, text=True) - if ret.returncode != 0: - print(ret.stdout) - print(ret.stderr) - die("CMake configuration failed") - - # Build + run(f"cmake -DCMAKE_BUILD_TYPE=Release {cmake_flags} ..", "CMake configuration failed") info("Running build...") - build_cmd = f"make -j$(nproc)" - ret = subprocess.run(build_cmd, shell=True, capture_output=True, text=True) - if ret.returncode != 0: - print(ret.stdout) - print(ret.stderr) - die("Build failed") - + run(f"make -j$(nproc)", "Build failed") info("Build completed successfully") - - # ============================================ - # Package 1: qt6-release-runtime - # ============================================ + + # Package 1: qt6-release-runtime (binaries and libraries, built via debuild) info("\n=== Building qt6-release-runtime ===") - pkg1_name = f"qt6-release-runtime" - pkg1_dir = os.path.join(build_base, f"{pkg1_name}-{version}") + pkg1_dir = os.path.join(build_base, f"qt6-release-runtime-{version}") os.makedirs(pkg1_dir) - - # Copy FULL source tree (needed for debian/rules cmake reference) - copy_src_dir_to_dest_dir(source_dir, pkg1_dir, release_scope=False) - - # Install lines for runtime: binaries and libraries from release scope only - # CMake install uses component prefixes: ALL->(none), AAR/SV/HLA->SV/, WS->WS/ - install_lines_1 = [] - - # Install from all possible component install paths - # ALL component: bin/, lib/ - # AAR/SV/HLA components: SV/bin/, SV/lib/ - # WS component: WS/bin/, WS/lib/ - install_lines_1.append("usr/bin/* usr/bin/") - install_lines_1.append("usr/lib/*.so* usr/lib/") - install_lines_1.append("usr/SV/bin/* usr/bin/") - install_lines_1.append("usr/SV/lib/*.so* usr/lib/") - install_lines_1.append("usr/WS/bin/* usr/bin/") - install_lines_1.append("usr/WS/lib/*.so* usr/lib/") - - build_single_deb( - pkg_name=pkg1_name, - pkg_dir=pkg1_dir, - output_dir=output_dir, - version=version, - build_number=build_number, - cmake_flags=cmake_flags, - skip_tests=True, - install_lines=install_lines_1, - description="Qt6 Release Runtime - binaries and libraries from release scope S", - section="libs" - ) - - # Create manifest - manifest_1 = create_manifest(output_dir, pkg1_name, pkg1_dir) - validate_package_contents(pkg1_name, manifest_1, + # FULL source tree (needed for debian/rules cmake reference) + copy_src_dir_to_dest_dir(source_dir, pkg1_dir) + # Install from all component install paths: ALL -> bin/, lib/ ; SV/ ; WS/ + install_lines_1 = [ + "usr/bin/* usr/bin/", + "usr/lib/*.so* usr/lib/", + "usr/SV/bin/* usr/bin/", + "usr/SV/lib/*.so* usr/lib/", + "usr/WS/bin/* usr/bin/", + "usr/WS/lib/*.so* usr/lib/", + ] + build_deb(pkg_name="qt6-release-runtime", pkg_dir=pkg1_dir, output_dir=output_dir, + version=version, build_number=build_number, cmake_flags=cmake_flags, + skip_tests=True, install_lines=install_lines_1, + description="Qt6 Release Runtime - binaries and libraries from release scope S", + section="libs") + manifest_1 = create_manifest(output_dir, "qt6-release-runtime", pkg1_dir) + validate_package_contents("qt6-release-runtime", manifest_1, allowed_patterns=["usr/bin/*", "usr/lib/*", "usr/SV/*", "usr/WS/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["*.cpp", "*.h", "*.hpp", "tests/*", "jenkins/*", "Dockerfile*"]) - - # ============================================ - # Package 2: qt6-release-sources - # ============================================ - info("\n=== Building qt6-release-sources ===") - pkg2_name = f"qt6-release-sources" - pkg2_dir = os.path.join(build_base, f"{pkg2_name}-{version}") - os.makedirs(pkg2_dir) - - # Copy FULL source tree (excluding DDS-related files) - copy_src_dir_to_dest_dir(source_dir, pkg2_dir, release_scope=False, file_filter=is_dds_file) - - # Install lines for sources: only from release scope directories - install_lines_2 = [] - for rel_dir in release_dirs: - src_dir = os.path.join(pkg2_dir, rel_dir) - if os.path.exists(src_dir): - for root, dirs, files in os.walk(src_dir): - dirs[:] = [d for d in dirs if not d.startswith('.')] - for file in files: - ext = os.path.splitext(file)[1].lower() - if ext in source_extensions or file == "CMakeLists.txt": - rel_path = os.path.relpath(os.path.join(root, file), pkg2_dir) - dest_dir = os.path.join("usr/src", os.path.dirname(rel_path)) - install_lines_2.append(f"{rel_path} {dest_dir}/") - - # Use no-build rules for source package - build_single_deb_nobuild( - pkg_name=pkg2_name, - pkg_dir=pkg2_dir, - output_dir=output_dir, - version=version, - build_number=build_number, - install_lines=install_lines_2, + + # Package 2: qt6-release-sources (release-scope sources, DDS excluded) + build_nobuild_package( + "qt6-release-sources", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release Sources - C++ source files from release scope S", - section="devel" - ) - - manifest_2 = create_manifest(output_dir, pkg2_name, pkg2_dir) - validate_package_contents(pkg2_name, manifest_2, - # scope filtering happens at staging time (release_dirs); the - # validator only guards against out-of-scope contamination + prepare=lambda pkg_dir: source_install_lines(pkg_dir, release_dirs), allowed_patterns=["usr/src/*", "usr/share/doc/*", "debian/*"], - forbidden_patterns=["build/*", "tests/*", "jenkins/*"]) - - # ============================================ - # Package 3: qt6-release-ctest-runtime - # ============================================ - info("\n=== Building qt6-release-ctest-runtime ===") - pkg3_name = f"qt6-release-ctest-runtime" - pkg3_dir = os.path.join(build_base, f"{pkg3_name}-{version}") - os.makedirs(pkg3_dir) - - # Copy FULL source tree - copy_src_dir_to_dest_dir(source_dir, pkg3_dir, release_scope=False) - - # Install lines for CTest runtime: test executables from release scope only - install_lines_3 = [] - - # Find test executables in build4/bin/ and map to release scope modules - test_executables = find_test_executables(pkg3_dir, release_dirs) - for exe_path, module_path in test_executables: - rel_path = os.path.relpath(exe_path, pkg3_dir) - dest_dir = "usr/lib/ctest" - install_lines_3.append(f"{rel_path} {dest_dir}/") - info(f" Including test executable: {os.path.basename(exe_path)} from {module_path}") - - # Also include CTest configuration files - ctest_files = [ - "build4/CTestTestfile.cmake", - "build4/DartConfiguration.tcl", - "build4/Testing/TAG", - ] - for ctest_file in ctest_files: - if os.path.exists(os.path.join(pkg3_dir, ctest_file)): - install_lines_3.append(f"{ctest_file} usr/lib/ctest/") - - build_single_deb_nobuild( - pkg_name=pkg3_name, - pkg_dir=pkg3_dir, - output_dir=output_dir, - version=version, - build_number=build_number, - install_lines=install_lines_3, + forbidden_patterns=["build/*", "tests/*", "jenkins/*"], + file_filter=is_dds_file) + + # Package 3: qt6-release-ctest-runtime (test executables from release scope) + build_nobuild_package( + "qt6-release-ctest-runtime", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release CTest Runtime - executable test programs from release scope", - section="devel", - skip_shlibdeps=True - ) - - manifest_3 = create_manifest(output_dir, pkg3_name, pkg3_dir) - validate_package_contents(pkg3_name, manifest_3, + prepare=lambda pkg_dir: ctest_runtime_install_lines(pkg_dir, release_dirs), allowed_patterns=["usr/lib/ctest/*", "usr/share/doc/*", "debian/*"], - forbidden_patterns=["*.cpp", "*.h", "jenkins/*"]) - - # ============================================ - # Package 4: qt6-release-ctest-sources - # ============================================ - info("\n=== Building qt6-release-ctest-sources ===") - pkg4_name = f"qt6-release-ctest-sources" - pkg4_dir = os.path.join(build_base, f"{pkg4_name}-{version}") - os.makedirs(pkg4_dir) - - # Copy FULL source tree - copy_src_dir_to_dest_dir(source_dir, pkg4_dir, release_scope=False) - - # Install lines for test sources from release scope only - install_lines_4 = [] - - # Find test source files in release scope modules - test_sources = find_test_sources(pkg4_dir, release_dirs) - for rel_path in test_sources: - dest_dir = os.path.dirname(rel_path) - install_lines_4.append(f"{rel_path} {dest_dir}/") - - build_single_deb_nobuild( - pkg_name=pkg4_name, - pkg_dir=pkg4_dir, - output_dir=output_dir, - version=version, - build_number=build_number, - install_lines=install_lines_4, + forbidden_patterns=["*.cpp", "*.h", "jenkins/*"], + skip_shlibdeps=True) + + # Package 4: qt6-release-ctest-sources (test source files from release scope) + build_nobuild_package( + "qt6-release-ctest-sources", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release CTest Sources - test source files from release scope", - section="devel" - ) - - manifest_4 = create_manifest(output_dir, pkg4_name, pkg4_dir) - validate_package_contents(pkg4_name, manifest_4, + prepare=lambda pkg_dir: test_sources_install_lines(pkg_dir, release_dirs), allowed_patterns=["Lts/*", "TetraIF/*", "Z/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["build/*", "jenkins/*"]) - - # ============================================ - # Package 5: qt6-release-build-pipeline - # ============================================ - info("\n=== Building qt6-release-build-pipeline ===") - pkg5_name = f"qt6-release-build-pipeline" - pkg5_dir = os.path.join(build_base, f"{pkg5_name}-{version}") - os.makedirs(pkg5_dir) - - # Copy FULL source tree - copy_src_dir_to_dest_dir(source_dir, pkg5_dir, release_scope=False) - - # Install lines for pipeline files only - install_lines_5 = [] - pipeline_files = [ - "new_docker_build.sh", - "Dockerfile", - "Dockerfile.ubuntu26", - "jenkins-ci-rme/Jenkinsfile", - "jenkins-ci-rme/jenkins_builder_ubuntu26.sh", - "jenkins-ci-rme/jenkins_publisher_ubuntu26.sh", - "jenkins-ci-rme/release-16-10-2026.sh", - "rpmbuild/make_rpm_ubuntu26.py", - "CMakeLists.txt", - "cmake/FindGeographicLib.cmake", - "cmake/FindRTI.cmake", - "cmake/FindZLIB.cmake", - ] - - for pipeline_file in pipeline_files: - src_file = os.path.join(pkg5_dir, pipeline_file) - if os.path.exists(src_file): - dest_dir = os.path.join("usr/src/pipeline", os.path.dirname(pipeline_file)) - install_lines_5.append(f"{pipeline_file} {dest_dir}/") - info(f" Added pipeline file: {pipeline_file}") - - build_single_deb_nobuild( - pkg_name=pkg5_name, - pkg_dir=pkg5_dir, - output_dir=output_dir, - version=version, - build_number=build_number, - install_lines=install_lines_5, + + # Package 5: qt6-release-build-pipeline (Docker, Jenkins, CMake, packaging scripts) + build_nobuild_package( + "qt6-release-build-pipeline", source_dir, build_base, output_dir, version, build_number, description="Qt6 Release Build Pipeline - Docker, Jenkins, CMake, and packaging scripts", - section="devel" - ) - - manifest_5 = create_manifest(output_dir, pkg5_name, pkg5_dir) - validate_package_contents(pkg5_name, manifest_5, + prepare=pipeline_install_lines, allowed_patterns=["usr/src/pipeline/*", "usr/share/doc/*", "debian/*"], forbidden_patterns=["*.cpp", "*.h", "tests/*"]) - + # Cleanup os.chdir(source_dir) remove_directory(build_base) - + info("\n========================================") info(f"DEB package build completed: {output_dir}") info("Generated packages:") @@ -896,28 +510,16 @@ def main(): parser.add_argument("--cmake_build_flags", default="", help="CMake build flags") parser.add_argument("--build_number", default="1", help="Build number") parser.add_argument("--release_scope", action="store_true", help="Only include release scope files") - + args = parser.parse_args() - info(f"Building {args.config} package for {args.project_name} v{args.version}") info(f"Output directory: {args.output_dir}") info(f"Source directory: {args.source_dir}") info(f"CMake flags: {args.cmake_build_flags}") info(f"Release scope: {args.release_scope}") - - source_dir = args.source_dir - - info(f"Source directory: {source_dir}") - - build_all_debs( - source_dir=source_dir, - output_dir=args.output_dir, - project_name=args.project_name, - version=args.version, - build_number=args.build_number, - cmake_flags=args.cmake_build_flags, - release_scope=args.release_scope - ) + build_all_debs(source_dir=args.source_dir, output_dir=args.output_dir, + version=args.version, build_number=args.build_number, + cmake_flags=args.cmake_build_flags) if __name__ == "__main__":