diff --git a/make_rpm_ubuntu26.py b/make_rpm_ubuntu26.py index bdc2f7c..fff3884 100755 --- a/make_rpm_ubuntu26.py +++ b/make_rpm_ubuntu26.py @@ -1,8 +1,13 @@ #!/usr/bin/python3 # -*- coding: utf-8 -*- """ -DEB package builder for Ubuntu 26.04 -Migrated from make_rpm.py (openSUSE) to Ubuntu/DEB ecosystem +DEB package builder for Ubuntu 26.04 - Qt6 Release +Creates 5 separate DEB packages as per customer requirements: +1. qt6-release-runtime - binaries and libraries from S +2. qt6-release-sources - Qt6 C++ sources from S +3. qt6-release-ctest-runtime - CTest executables +4. qt6-release-ctest-sources - CTest source files +5. qt6-release-build-pipeline - Dockerfiles, build scripts, CI/CD files """ import argparse @@ -51,210 +56,239 @@ def die(message): sys.exit(1) -def system(cmdstring, file=None): - debug(f"Executing '{cmdstring}'") - stdout = subprocess.PIPE - if file is not None: - stdout = file - p = subprocess.Popen(cmdstring, shell=True, stdout=stdout, stderr=sys.stderr) - ret = p.wait() - if ret != 0: - die(f"Error executing {cmdstring} ({ret})") - ret = "" - if file is None: - ret = p.stdout.read().strip() - return ret +def remove_directory(path): + """Remove directory tree""" + if os.path.exists(path): + shutil.rmtree(path) -def get_os_name(): - """Return Ubuntu version name""" - if os.path.exists("/etc/os-release"): - with open("/etc/os-release", "r") as f: - for line in f.readlines(): - if line.startswith("PRETTY_NAME="): - name = line.split("=")[1].strip().strip('"') - return name.replace(" ", "_") - return "Ubuntu_unknown" +def copy_file_to_dest_dir(src_file, dest_dir): + """Copy a single file to destination directory""" + os.makedirs(dest_dir, exist_ok=True) + shutil.copy2(src_file, dest_dir) -def remove_directory(dest_dir): - info("--------------------------------") - info(f"Removing {dest_dir}") - info("--------------------------------") - if os.path.exists(dest_dir) and os.path.isdir(dest_dir): - shutil.rmtree(dest_dir) - - -def copy_src_dir_to_dest_dir(srcdir, dstdir, perm=None, with_subdirs=True, release_scope=False): - """Copy source directory to destination directory. +def is_dds_file(filepath): + """Check if a file or directory path contains DDS-related content that should be excluded Args: - srcdir: Source directory path - dstdir: Destination directory path - perm: Optional permissions to set - with_subdirs: Whether to copy subdirectories - release_scope: If True, only copy release scope files (TetraIF, Z/src, Lts/Common) + filepath: Path to check + + Returns: + True if file should be excluded (is DDS-related), False otherwise """ - info("--------------------------------") - info(f"Copying {srcdir} to {dstdir}") - if release_scope: - info("Release scope mode: only copying TetraIF, Z/src, Lts/Common") - info("--------------------------------") - - while srcdir.endswith("/"): - srcdir = srcdir[:-1] + # Convert to lowercase for case-insensitive matching + path_lower = filepath.lower() - # Release scope directories and files to include - release_items = [ - 'TetraIF', - 'Z/src', - 'Lts/Common', - 'Lts/Proxies/TrainingAreaBase', # Required for training_area_telegrams.h used by TetraIF - 'Lts/Services/ProjectDataManagement/stubs', # Stub headers for pdm.hpp (needed by TrainingAreaBase) - 'CMakeLists.txt', - 'cmake', - 'tests', - 'new_docker_build.sh', - 'specs', - 'translations', - 'CTestCustom.ctest', - '.gitignore', - '.envrc', - 'doxygen', - 'external', - 'src', - 'lts-cc.conf' + # Exclude files/directories with 'dds' in the name + if 'dds' in path_lower: + return True + + return False + + +def copy_src_dir_to_dest_dir(src_dir, dest_dir, release_scope=False, file_filter=None): + """Copy source directory to destination with optional filtering + + Args: + src_dir: Source directory + dest_dir: Destination directory + release_scope: If True, only copy release scope directories + file_filter: Optional function to filter files (filepath) -> bool + """ + os.makedirs(dest_dir, exist_ok=True) + + # Release scope directories + release_dirs = [ + "TetraIF", + "Z/src", + "Lts/Common" ] - # Directories to exclude from include folder (bundled libraries not needed for release) - include_exclude = ['GeographicLib'] - - if with_subdirs: - for root, dirs, files in os.walk(srcdir): - # Filter directories for release scope - if release_scope: - rel_root = os.path.relpath(root, srcdir) - if rel_root == '.': - # At root level, only keep release scope directories - # Need to handle multi-level paths like 'Lts/Proxies/TrainingAreaBase' - def should_keep_dir(d): - # Check if directory is directly in release_items - if d in release_items: - return True - # Check if directory is the first part of a multi-level release item - for item in release_items: - if '/' in item: - first_part = item.split('/')[0] - if d == first_part: - return True - return False - kept_dirs = [d for d in dirs if should_keep_dir(d)] - info(f"Root level: keeping {kept_dirs} from {dirs}") - dirs[:] = kept_dirs - elif rel_root == 'Z': - # In Z directory, only keep src - dirs[:] = [d for d in dirs if d == 'src'] - elif rel_root == 'Lts': - # In Lts directory, only keep Common, Proxies (for TrainingAreaBase), and Services (for ProjectDataManagement stubs) - kept_dirs = [d for d in dirs if d in ('Common', 'Proxies', 'Services')] - info(f"Lts level: keeping {kept_dirs} from {dirs}") - dirs[:] = kept_dirs - elif rel_root == 'Lts/Proxies': - # In Lts/Proxies, only keep TrainingAreaBase - kept_dirs = [d for d in dirs if d == 'TrainingAreaBase'] - info(f"Lts/Proxies level: keeping {kept_dirs} from {dirs}") - dirs[:] = kept_dirs - elif rel_root.startswith('Lts/Proxies/TrainingAreaBase'): - # Keep all subdirectories under TrainingAreaBase (DDS code wrapped in conditionals) - pass - elif rel_root == 'Lts/Services': - # In Lts/Services, only keep ProjectDataManagement (for stub headers) - kept_dirs = [d for d in dirs if d == 'ProjectDataManagement'] - info(f"Lts/Services level: keeping {kept_dirs} from {dirs}") - dirs[:] = kept_dirs - elif rel_root == 'Lts/Services/ProjectDataManagement': - # In ProjectDataManagement, only keep stubs - kept_dirs = [d for d in dirs if d == 'stubs'] - info(f"Lts/Services/ProjectDataManagement level: keeping {kept_dirs} from {dirs}") - dirs[:] = kept_dirs - elif rel_root.startswith('Lts/Services/ProjectDataManagement/stubs'): - # Keep all subdirectories under stubs (contains ProjectDataManagement/pdm.hpp) - pass - elif rel_root.startswith('Lts/Common'): - # Keep all subdirectories under Lts/Common except LtsUtilsGuiDds (requires DDS) - if rel_root == 'Lts/Common/LtsUtils/LtsUtilsGui': - dirs[:] = [d for d in dirs if d != 'LtsUtilsGuiDds'] - else: - pass - elif rel_root.startswith('Z/src'): - # Keep all subdirectories under Z/src - pass - elif rel_root.startswith('TetraIF'): - # Keep all subdirectories under TetraIF - pass - elif rel_root == 'include': - # In include directory, exclude bundled libraries - dirs[:] = [d for d in dirs if d not in include_exclude] - else: - # Skip other directories - dirs[:] = [] - - tmpdir = root.replace(srcdir, dstdir) - debug(f"dstdir = '{tmpdir}'") - if not os.path.exists(tmpdir): - debug(f"Creating dir '{tmpdir}'") - os.makedirs(tmpdir) - if perm: - os.chmod(tmpdir, perm | stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) - for file in files: - # Skip non-release files at root level - if release_scope and rel_root == '.': - if file not in release_items and not any(file == item.split('/')[0] for item in release_items if '/' in item): - continue - # Include 'include' directory but GeographicLib will be filtered during walk - src_path = os.path.join(root, file) - # Skip broken symlinks - if os.path.islink(src_path) and not os.path.exists(src_path): - debug(f"Skipping broken symlink: {src_path}") - continue - if os.path.isfile(tmpdir) and os.path.exists(tmpdir): - os.remove(tmpdir) - shutil.copy(src_path, tmpdir, follow_symlinks=True) - if perm: - f = os.path.join(tmpdir, file) - add_perm = 0 - if os.path.isdir(f): - add_perm = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - else: - mode = os.stat(f).st_mode - if (stat.S_IXUSR & mode) == stat.S_IXUSR: - add_perm = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH - os.chmod(f, perm | add_perm) + if release_scope: + info(f"Copying only release scope directories: {release_dirs}") + # Also copy root-level build files needed for CMake + root_build_files = ['CMakeLists.txt', 'CTestCustom.ctest'] + root_build_dirs = ['cmake'] + + for bf in root_build_files: + src_file = os.path.join(src_dir, bf) + if os.path.exists(src_file): + debug(f" Copying root file {bf}...") + shutil.copy2(src_file, os.path.join(dest_dir, bf)) + + for bd in root_build_dirs: + src_bdir = os.path.join(src_dir, bd) + dest_bdir = os.path.join(dest_dir, bd) + if os.path.exists(src_bdir): + debug(f" Copying root directory {bd}...") + shutil.copytree(src_bdir, dest_bdir, dirs_exist_ok=True) + + for rel_dir in release_dirs: + src_subdir = os.path.join(src_dir, rel_dir) + dest_subdir = os.path.join(dest_dir, rel_dir) + if os.path.exists(src_subdir): + debug(f" Copying {rel_dir}...") + for root, dirs, files in os.walk(src_subdir): + # Skip build directories, hidden directories, deb_build_tmp (prevent recursion), and DDS-related directories + dirs[:] = [d for d in dirs if not d.startswith('.') + and d not in ['build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'] + and (file_filter is None or not file_filter(os.path.join(root, d)))] + + for file in files: + src_file = os.path.join(root, file) + rel_path = os.path.relpath(src_file, src_subdir) + dest_file = os.path.join(dest_subdir, rel_path) + + # Apply file filter if provided (skip file if filter returns True) + if file_filter and file_filter(src_file): + continue + + os.makedirs(os.path.dirname(dest_file), exist_ok=True) + shutil.copy2(src_file, dest_file) else: - for f in os.listdir(srcdir): - if os.path.isdir(f): - continue - if release_scope and f not in release_items: - continue - shutil.copy(os.path.join(srcdir, f), os.path.join(dstdir, f)) + # Copy entire source tree + for root, dirs, files in os.walk(src_dir): + # Skip build directories, hidden directories, deb_build_tmp (prevent recursion), and DDS-related directories + dirs[:] = [d for d in dirs if not d.startswith('.') + and d not in ['build', 'cmake-build-debug', 'cmake-build-release', 'deb_build_tmp'] + and (file_filter is None or not file_filter(os.path.join(root, d)))] + + for file in files: + src_file = os.path.join(root, file) + rel_path = os.path.relpath(src_file, src_dir) + dest_file = os.path.join(dest_dir, rel_path) + + # Apply file filter if provided (skip file if filter returns True) + if file_filter and file_filter(src_file): + continue + + os.makedirs(os.path.dirname(dest_file), exist_ok=True) + shutil.copy2(src_file, dest_file) -def create_debian_control(deb_dir, project_name, version, build_number, cmake_flags): - """Create debian/control file for DEB package""" - control_content = f"""Source: legatus-{project_name.lower()} -Section: devel +def find_test_executables(pkg_dir, release_dirs): + """Find test executables from release scope modules in build4/bin/ + + Args: + pkg_dir: Package staging directory + release_dirs: List of release scope directory paths + + Returns: + List of (executable_path, module_path) tuples for tests in release scope + """ + test_executables = [] + build_bin_dir = os.path.join(pkg_dir, "build4", "bin") + + if not os.path.exists(build_bin_dir): + warning(f"Build bin directory not found: {build_bin_dir}") + return test_executables + + # Get all executable files in build4/bin/ + for filename in os.listdir(build_bin_dir): + file_path = os.path.join(build_bin_dir, filename) + if not os.path.isfile(file_path) or not os.access(file_path, os.X_OK): + continue + + # Skip non-test executables (common patterns) + if not ('test' in filename.lower() or 'Test' in filename): + continue + + # Find which module this test belongs to by searching CMakeLists.txt + module_path = find_test_module_source(pkg_dir, filename, release_dirs) + if module_path: + test_executables.append((file_path, module_path)) + debug(f" Found test executable: {filename} from {module_path}") + else: + debug(f" Skipping test executable (no module found): {filename}") + + return test_executables + + +def find_test_module_source(pkg_dir, test_name, release_dirs): + """Find the source module directory for a test executable + + Args: + pkg_dir: Package staging directory + test_name: Name of the test executable + release_dirs: List of release scope directory paths + + Returns: + Relative path to module directory, or None if not found + """ + # Search for CMakeLists.txt files that define this test + for rel_dir in release_dirs: + module_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(module_dir): + continue + + for root, dirs, files in os.walk(module_dir): + # Skip build directories + dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['build', 'cmake-build-debug', 'cmake-build-release']] + + if 'CMakeLists.txt' in files: + cmake_file = os.path.join(root, 'CMakeLists.txt') + try: + with open(cmake_file, 'r') as f: + content = f.read() + # Look for ADD_EXECUTABLE or ADD_TEST with this test name + if f'ADD_EXECUTABLE({test_name}' in content or f'ADD_TEST(NAME {test_name}' in content: + # Return the module directory relative to pkg_dir + return os.path.relpath(root, pkg_dir) + except Exception: + pass + + return None + + +def find_test_sources(pkg_dir, release_dirs): + """Find test source files in release scope modules + + Args: + pkg_dir: Package staging directory + release_dirs: List of release scope directory paths + + Returns: + List of relative paths to test source files + """ + test_sources = [] + test_patterns = ['*test*.cpp', '*test*.h', '*Test*.cpp', '*Test*.h', 'test_*.cpp', 'test_*.h'] + + for rel_dir in release_dirs: + module_dir = os.path.join(pkg_dir, rel_dir) + if not os.path.exists(module_dir): + continue + + for root, dirs, files in os.walk(module_dir): + # Skip build directories and hidden directories + dirs[:] = [d for d in dirs if not d.startswith('.') and d not in ['build', 'cmake-build-debug', 'cmake-build-release']] + + for filename in files: + # Check if this is a test file + if 'test' in filename.lower() and (filename.endswith('.cpp') or filename.endswith('.h')): + file_path = os.path.join(root, filename) + rel_path = os.path.relpath(file_path, pkg_dir) + test_sources.append(rel_path) + debug(f" Found test source: {rel_path}") + + return test_sources + + +def create_debian_control(deb_dir, package_name, version, build_number, description, depends="", section="devel"): + """Create debian/control file for a package""" + control_content = f"""Source: {package_name} +Section: {section} Priority: optional -Maintainer: Legatus Team -Build-Depends: debhelper (>= 13), cmake, ninja-build, qt6-base-dev, qt6-tools-dev, - libboost-all-dev, libjson-glib-dev, libgeographiclib-dev +Maintainer: Legatus Team +Build-Depends: debhelper (>= 13), cmake, qt6-base-dev, qt6-tools-dev +Standards-Version: 4.6.0 -Package: legatus-{project_name.lower()} +Package: {package_name} Architecture: any -Depends: ${{shlibs:Depends}}, ${{misc:Depends}} -Description: Legatus {project_name} LTS package - Legatus {project_name} component built with Qt6/C++23 - Version: {version}-{build_number} - Build flags: {cmake_flags} +Depends: ${{shlibs:Depends}}, ${{misc:Depends}}{", " + depends if depends else ""} +Description: {description} + Legatus Qt6 Release Package + This package is part of the Legatus Qt6 migration release. """ control_file = os.path.join(deb_dir, "control") @@ -263,15 +297,16 @@ Description: Legatus {project_name} LTS package info(f"Created {control_file}") -def create_debian_changelog(deb_dir, project_name, version, build_number): +def create_debian_changelog(deb_dir, package_name, version, build_number): """Create debian/changelog file""" - timestamp = datetime.now().strftime("%a, %d %b %Y %H:%M:%S +0000") - changelog_content = f"""legatus-{project_name.lower()} ({version}-{build_number}) unstable; urgency=medium + # Use +0000 as timezone offset (UTC) for reproducibility + from email.utils import formatdate + timestamp = formatdate(localtime=True) + changelog_content = f"""{package_name} ({version}-{build_number}) unstable; urgency=medium - * Automated build for Ubuntu 26.04 - * Qt6/C++23 migration + * Qt6 Release Build - -- Legatus Team {timestamp} + -- Legatus Team {timestamp} """ changelog_file = os.path.join(deb_dir, "changelog") @@ -280,44 +315,27 @@ def create_debian_changelog(deb_dir, project_name, version, build_number): info(f"Created {changelog_file}") -def create_debian_rules(deb_dir, cmake_flags="", release_scope=False): - """Create debian/rules file - - Args: - deb_dir: debian directory path - cmake_flags: Additional CMake flags to pass - release_scope: If True, skip tests for release build - """ - # Build CMake flags string - cmake_opts = "-DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr" - if cmake_flags: - cmake_opts = f"{cmake_opts} {cmake_flags}" - - # For release builds, skip tests to avoid test infrastructure issues +def create_debian_rules(deb_dir, cmake_flags="", skip_tests=False): + """Create debian/rules file""" test_override = "" - if release_scope: + if skip_tests: test_override = """ override_dh_auto_test: - # Skip tests for release build - tests run separately in CI - @echo "Skipping tests for release build" + # Tests skipped for this package type """ rules_content = f"""#!/usr/bin/make -f %: - dh $@ --buildsystem=cmake --builddirectory=build - -override_dh_clean: - dh_clean - rm -rf build + dh $@ --buildsystem=cmake override_dh_auto_configure: - dh_auto_configure -- {cmake_opts} + dh_auto_configure -- -DCMAKE_BUILD_TYPE=Release {cmake_flags} override_dh_auto_build: dh_auto_build --parallel override_dh_auto_install: - dh_auto_install + dh_auto_install --destdir=debian/tmp {test_override}""" rules_file = os.path.join(deb_dir, "rules") @@ -327,54 +345,153 @@ override_dh_auto_install: info(f"Created {rules_file}") -def build_deb(source_dir, output_dir, project_name, version, build_number, cmake_flags, release_scope=False): - """Build DEB package using debuild +def create_debian_rules_nobuild(deb_dir, skip_shlibdeps=False): + """Create debian/rules file for packages that don't need building (pre-built files only) Args: - source_dir: Source directory path - output_dir: Output directory for packages - project_name: Project name (qatar|ahtc|guez) - version: Version string - build_number: Build number - cmake_flags: CMake build flags - release_scope: If True, only include release scope files (TetraIF, Z/src, Lts/Common) + deb_dir: Path to debian directory + skip_shlibdeps: If True, skip dh_shlibdeps (useful for test executables with external deps) """ - info("--------------------------------") - info(f"Building DEB package for {project_name} {version}-{build_number}") - if release_scope: - info("Release scope mode: only packaging TetraIF, Z/src, Lts/Common") - info("--------------------------------") + rules_content = """#!/usr/bin/make -f +%: + dh $@ + +override_dh_auto_build: + # No build step needed - pre-built files only + +override_dh_auto_configure: + # No configure step needed - pre-built files only + +override_dh_auto_install: + dh_auto_install +""" + if skip_shlibdeps: + rules_content += """ +override_dh_shlibdeps: + # Skip shlibdeps for test executables - Qt6 deps are external system packages +""" - # Create output directory - os.makedirs(output_dir, exist_ok=True) + rules_file = os.path.join(deb_dir, "rules") + with open(rules_file, "w") as f: + f.write(rules_content) + os.chmod(rules_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH) + info(f"Created {rules_file}") - # Create temporary build directory - build_base = os.path.join(source_dir, "deb_build_tmp") - remove_directory(build_base) - os.makedirs(build_base) + rules_file = os.path.join(deb_dir, "rules") + with open(rules_file, "w") as f: + f.write(rules_content) + os.chmod(rules_file, stat.S_IREAD | stat.S_IWRITE | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH | stat.S_IXGRP | stat.S_IXOTH) + info(f"Created {rules_file}") + + +def create_debian_install(deb_dir, install_lines): + """Create debian/install file from list of file entries""" + install_file = os.path.join(deb_dir, "install") + with open(install_file, "w") as f: + for line in install_lines: + f.write(line + "\n") + info(f"Created {install_file} with {len(install_lines)} entries") + + +def create_manifest(output_dir, package_name, pkg_dir): + """Create manifest file listing all package contents from actual DEB file""" + manifest_file = os.path.join(output_dir, "manifests", f"{package_name}.manifest") + os.makedirs(os.path.dirname(manifest_file), exist_ok=True) - # Copy source to build directory - pkg_name = f"legatus-{project_name.lower()}" - pkg_dir = os.path.join(build_base, f"{pkg_name}-{version}") - copy_src_dir_to_dest_dir(source_dir, pkg_dir, release_scope=release_scope) + # Find the DEB file for this package + deb_file = None + for f in os.listdir(output_dir): + if f.startswith(package_name) and f.endswith(".deb"): + deb_file = os.path.join(output_dir, f) + break + + manifest_lines = [] + if deb_file and os.path.exists(deb_file): + # Extract DEB contents using dpkg-deb + result = subprocess.run( + ["dpkg-deb", "--contents", deb_file], + capture_output=True, text=True + ) + if result.returncode == 0: + for line in result.stdout.strip().split("\n"): + if line.strip(): + # dpkg-deb --contents output format: -rw-r--r-- root/root 1234 2024-01-01 00:00 ./path/to/file + # Extract the path (last field) + parts = line.split() + if len(parts) >= 6: + file_path = parts[-1] + # Remove leading ./ or / + if file_path.startswith("./"): + file_path = file_path[2:] + elif file_path.startswith("/"): + file_path = file_path[1:] + if file_path: + manifest_lines.append(file_path) + elif os.path.exists(pkg_dir): + # Fallback: scan directory if DEB not found + info(f"Warning: DEB file not found for {package_name}, scanning directory instead") + for root, dirs, files in os.walk(pkg_dir): + for file in files: + file_path = os.path.join(root, file) + rel_path = os.path.relpath(file_path, pkg_dir) + manifest_lines.append(rel_path) + + manifest_lines.sort() + with open(manifest_file, "w") as f: + for line in manifest_lines: + f.write(line + "\n") + + info(f"Created manifest: {manifest_file} ({len(manifest_lines)} entries)") + return manifest_lines + + +def validate_package_contents(package_name, manifest_lines, allowed_patterns, forbidden_patterns=None): + """Validate package contents against allowlist/forbidden list""" + import fnmatch + + errors = [] + warnings = [] + + for file_path in manifest_lines: + # Check forbidden patterns + if forbidden_patterns: + for pattern in forbidden_patterns: + if fnmatch.fnmatch(file_path, pattern): + errors.append(f"FORBIDDEN file in {package_name}: {file_path} (matches {pattern})") + + # Check if file matches any allowed pattern + allowed = False + if allowed_patterns: + for pattern in allowed_patterns: + if fnmatch.fnmatch(file_path, pattern): + allowed = True + break + + if not allowed and allowed_patterns: + warnings.append(f"Unexpected file in {package_name}: {file_path}") + + if errors: + for err in errors: + error(f"Validation error: {err}") + return False + + if warnings: + for warn in warnings: + warning(f"Validation warning: {warn}") + + return True + + +def build_single_deb(pkg_name, pkg_dir, output_dir, version, build_number, cmake_flags, skip_tests, install_lines, description, section="devel", depends=""): + """Build a single DEB package""" + info(f"Building DEB package: {pkg_name}") - # Create debian directory deb_dir = os.path.join(pkg_dir, "debian") os.makedirs(deb_dir) - # Create debian control files - create_debian_control(deb_dir, project_name, version, build_number, cmake_flags) - create_debian_changelog(deb_dir, project_name, version, build_number) - - # Add RELEASE_BUILD=ON for release scope builds - deb_cmake_flags = cmake_flags - if release_scope: - if deb_cmake_flags: - deb_cmake_flags = f"{deb_cmake_flags} -DRELEASE_BUILD=ON" - else: - deb_cmake_flags = "-DRELEASE_BUILD=ON" - - create_debian_rules(deb_dir, deb_cmake_flags, release_scope) + create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) + create_debian_changelog(deb_dir, pkg_name, version, build_number) + create_debian_rules(deb_dir, cmake_flags, skip_tests) # Create debian/source/format source_dir_path = os.path.join(deb_dir, "source") @@ -382,24 +499,29 @@ def build_deb(source_dir, output_dir, project_name, version, build_number, cmake with open(os.path.join(source_dir_path, "format"), "w") as f: f.write("3.0 (quilt)\n") - # Create debian/compat (for older debhelper compatibility) + # Create debian/compat with open(os.path.join(deb_dir, "compat"), "w") as f: f.write("13\n") + # Create debian/install + if install_lines: + create_debian_install(deb_dir, install_lines) + # Build the package - info("Running debuild...") + info(f"Running debuild for {pkg_name}...") os.chdir(pkg_dir) - # Set environment for debuild env = os.environ.copy() env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" cmd = f"debuild -us -uc -b" debug(f"Executing: {cmd}") - ret = subprocess.run(cmd, shell=True, env=env) + ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) if ret.returncode != 0: - die(f"debuild failed with code {ret.returncode}") + print(ret.stdout) + print(ret.stderr) + die(f"debuild failed for {pkg_name} with code {ret.returncode}") # Move generated packages to output directory parent_dir = os.path.dirname(pkg_dir) @@ -412,15 +534,351 @@ def build_deb(source_dir, output_dir, project_name, version, build_number, cmake shutil.move(src, output_dir) info(f"Moved {f} to {output_dir}") + return True + + +def build_single_deb_nobuild(pkg_name, pkg_dir, output_dir, version, build_number, install_lines, description, section="devel", depends="", skip_shlibdeps=False): + """Build a single DEB package without running cmake/make (pre-built files only) + + Args: + skip_shlibdeps: If True, skip dh_shlibdeps (useful for test executables with external deps) + """ + info(f"Building DEB package (no-build): {pkg_name}") + + deb_dir = os.path.join(pkg_dir, "debian") + os.makedirs(deb_dir) + + create_debian_control(deb_dir, pkg_name, version, build_number, description, depends, section) + create_debian_changelog(deb_dir, pkg_name, version, build_number) + create_debian_rules_nobuild(deb_dir, skip_shlibdeps=skip_shlibdeps) + + # Create debian/source/format + source_dir_path = os.path.join(deb_dir, "source") + os.makedirs(source_dir_path) + with open(os.path.join(source_dir_path, "format"), "w") as f: + f.write("3.0 (quilt)\n") + + # Create debian/compat + with open(os.path.join(deb_dir, "compat"), "w") as f: + f.write("13\n") + + # Create debian/install + if install_lines: + create_debian_install(deb_dir, install_lines) + + # Build the package + info(f"Running debuild for {pkg_name}...") + os.chdir(pkg_dir) + + env = os.environ.copy() + env["DEB_BUILD_OPTIONS"] = "parallel=$(nproc)" + + cmd = f"debuild -us -uc -b" + debug(f"Executing: {cmd}") + ret = subprocess.run(cmd, shell=True, env=env, capture_output=True, text=True) + + if ret.returncode != 0: + print(ret.stdout) + print(ret.stderr) + die(f"debuild failed for {pkg_name} with code {ret.returncode}") + + # Move generated packages to output directory + parent_dir = os.path.dirname(pkg_dir) + for f in os.listdir(parent_dir): + if f.endswith(".deb") or f.endswith(".dsc") or f.endswith(".changes") or f.endswith(".buildinfo"): + src = os.path.join(parent_dir, f) + dst = os.path.join(output_dir, f) + if os.path.exists(dst): + os.remove(dst) + shutil.move(src, output_dir) + info(f"Moved {f} to {output_dir}") + + return True + + +def build_all_debs(source_dir, output_dir, project_name, version, build_number, cmake_flags, release_scope=True): + """Build all 5 required DEB packages - build once, then package""" + info("========================================") + info("Building 5 DEB packages for Qt6 Release") + info("========================================") + + os.makedirs(output_dir, exist_ok=True) + os.makedirs(os.path.join(output_dir, "manifests"), exist_ok=True) + + build_base = os.path.join(source_dir, "deb_build_tmp") + remove_directory(build_base) + os.makedirs(build_base) + + # Release scope directories + release_dirs = ["TetraIF", "Z/src", "Lts/Common"] + + # Source file extensions + source_extensions = ['.cpp', '.h', '.hpp', '.cc', '.cxx', '.hxx'] + script_extensions = ['.py', '.sh', '.cmake', '.txt'] + + # ============================================ + # STEP 1: Build once from full source tree + # ============================================ + info("\n=== Step 1: Building from full source tree ===") + build_dir = os.path.join(source_dir, "build") + os.makedirs(build_dir, exist_ok=True) + os.chdir(build_dir) + + # Configure + info("Running CMake configuration...") + cmake_cmd = f"cmake -DCMAKE_BUILD_TYPE=Release {cmake_flags} .." + ret = subprocess.run(cmake_cmd, shell=True, capture_output=True, text=True) + if ret.returncode != 0: + print(ret.stdout) + print(ret.stderr) + die("CMake configuration failed") + + # Build + info("Running build...") + build_cmd = f"make -j$(nproc)" + ret = subprocess.run(build_cmd, shell=True, capture_output=True, text=True) + if ret.returncode != 0: + print(ret.stdout) + print(ret.stderr) + die("Build failed") + + info("Build completed successfully") + + # ============================================ + # Package 1: qt6-release-runtime + # ============================================ + info("\n=== Building qt6-release-runtime ===") + pkg1_name = f"qt6-release-runtime" + pkg1_dir = os.path.join(build_base, f"{pkg1_name}-{version}") + os.makedirs(pkg1_dir) + + # Copy FULL source tree (needed for debian/rules cmake reference) + copy_src_dir_to_dest_dir(source_dir, pkg1_dir, release_scope=False) + + # Install lines for runtime: binaries and libraries from release scope only + # CMake install uses component prefixes: ALL->(none), AAR/SV/HLA->SV/, WS->WS/ + install_lines_1 = [] + + # Install from all possible component install paths + # ALL component: bin/, lib/ + # AAR/SV/HLA components: SV/bin/, SV/lib/ + # WS component: WS/bin/, WS/lib/ + install_lines_1.append("usr/bin/* usr/bin/") + install_lines_1.append("usr/lib/*.so* usr/lib/") + install_lines_1.append("usr/SV/bin/* usr/bin/") + install_lines_1.append("usr/SV/lib/*.so* usr/lib/") + install_lines_1.append("usr/WS/bin/* usr/bin/") + install_lines_1.append("usr/WS/lib/*.so* usr/lib/") + + build_single_deb( + pkg_name=pkg1_name, + pkg_dir=pkg1_dir, + output_dir=output_dir, + version=version, + build_number=build_number, + cmake_flags=cmake_flags, + skip_tests=True, + install_lines=install_lines_1, + description="Qt6 Release Runtime - binaries and libraries from release scope S", + section="libs" + ) + + # Create manifest + manifest_1 = create_manifest(output_dir, pkg1_name, pkg1_dir) + validate_package_contents(pkg1_name, manifest_1, + allowed_patterns=["usr/bin/*", "usr/lib/*", "usr/SV/*", "usr/WS/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "*.hpp", "tests/*", "jenkins/*", "Dockerfile*"]) + + # ============================================ + # Package 2: qt6-release-sources + # ============================================ + info("\n=== Building qt6-release-sources ===") + pkg2_name = f"qt6-release-sources" + pkg2_dir = os.path.join(build_base, f"{pkg2_name}-{version}") + os.makedirs(pkg2_dir) + + # Copy FULL source tree (excluding DDS-related files) + copy_src_dir_to_dest_dir(source_dir, pkg2_dir, release_scope=False, file_filter=is_dds_file) + + # Install lines for sources: only from release scope directories + install_lines_2 = [] + for rel_dir in release_dirs: + src_dir = os.path.join(pkg2_dir, rel_dir) + if os.path.exists(src_dir): + for root, dirs, files in os.walk(src_dir): + dirs[:] = [d for d in dirs if not d.startswith('.')] + for file in files: + ext = os.path.splitext(file)[1].lower() + if ext in source_extensions or file == "CMakeLists.txt": + rel_path = os.path.relpath(os.path.join(root, file), pkg2_dir) + dest_dir = os.path.join("usr/src", os.path.dirname(rel_path)) + install_lines_2.append(f"{rel_path} {dest_dir}/") + + # Use no-build rules for source package + build_single_deb_nobuild( + pkg_name=pkg2_name, + pkg_dir=pkg2_dir, + output_dir=output_dir, + version=version, + build_number=build_number, + install_lines=install_lines_2, + description="Qt6 Release Sources - C++ source files from release scope S", + section="devel" + ) + + manifest_2 = create_manifest(output_dir, pkg2_name, pkg2_dir) + validate_package_contents(pkg2_name, manifest_2, + allowed_patterns=["usr/src/**/*.cpp", "usr/src/**/*.h", "usr/src/**/CMakeLists.txt", "debian/*"], + forbidden_patterns=["build/*", "tests/*", "jenkins/*"]) + + # ============================================ + # Package 3: qt6-release-ctest-runtime + # ============================================ + info("\n=== Building qt6-release-ctest-runtime ===") + pkg3_name = f"qt6-release-ctest-runtime" + pkg3_dir = os.path.join(build_base, f"{pkg3_name}-{version}") + os.makedirs(pkg3_dir) + + # Copy FULL source tree + copy_src_dir_to_dest_dir(source_dir, pkg3_dir, release_scope=False) + + # Install lines for CTest runtime: test executables from release scope only + install_lines_3 = [] + + # Find test executables in build4/bin/ and map to release scope modules + test_executables = find_test_executables(pkg3_dir, release_dirs) + for exe_path, module_path in test_executables: + rel_path = os.path.relpath(exe_path, pkg3_dir) + dest_dir = "usr/lib/ctest" + install_lines_3.append(f"{rel_path} {dest_dir}/") + info(f" Including test executable: {os.path.basename(exe_path)} from {module_path}") + + # Also include CTest configuration files + ctest_files = [ + "build4/CTestTestfile.cmake", + "build4/DartConfiguration.tcl", + "build4/Testing/TAG", + ] + for ctest_file in ctest_files: + if os.path.exists(os.path.join(pkg3_dir, ctest_file)): + install_lines_3.append(f"{ctest_file} usr/lib/ctest/") + + build_single_deb_nobuild( + pkg_name=pkg3_name, + pkg_dir=pkg3_dir, + output_dir=output_dir, + version=version, + build_number=build_number, + install_lines=install_lines_3, + description="Qt6 Release CTest Runtime - executable test programs from release scope", + section="devel", + skip_shlibdeps=True + ) + + manifest_3 = create_manifest(output_dir, pkg3_name, pkg3_dir) + validate_package_contents(pkg3_name, manifest_3, + allowed_patterns=["usr/lib/ctest/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "jenkins/*"]) + + # ============================================ + # Package 4: qt6-release-ctest-sources + # ============================================ + info("\n=== Building qt6-release-ctest-sources ===") + pkg4_name = f"qt6-release-ctest-sources" + pkg4_dir = os.path.join(build_base, f"{pkg4_name}-{version}") + os.makedirs(pkg4_dir) + + # Copy FULL source tree + copy_src_dir_to_dest_dir(source_dir, pkg4_dir, release_scope=False) + + # Install lines for test sources from release scope only + install_lines_4 = [] + + # Find test source files in release scope modules + test_sources = find_test_sources(pkg4_dir, release_dirs) + for rel_path in test_sources: + dest_dir = os.path.dirname(rel_path) + install_lines_4.append(f"{rel_path} {dest_dir}/") + + build_single_deb_nobuild( + pkg_name=pkg4_name, + pkg_dir=pkg4_dir, + output_dir=output_dir, + version=version, + build_number=build_number, + install_lines=install_lines_4, + description="Qt6 Release CTest Sources - test source files from release scope", + section="devel" + ) + + manifest_4 = create_manifest(output_dir, pkg4_name, pkg4_dir) + validate_package_contents(pkg4_name, manifest_4, + allowed_patterns=["usr/src/**/tests/*", "debian/*"], + forbidden_patterns=["build/*", "jenkins/*"]) + + # ============================================ + # Package 5: qt6-release-build-pipeline + # ============================================ + info("\n=== Building qt6-release-build-pipeline ===") + pkg5_name = f"qt6-release-build-pipeline" + pkg5_dir = os.path.join(build_base, f"{pkg5_name}-{version}") + os.makedirs(pkg5_dir) + + # Copy FULL source tree + copy_src_dir_to_dest_dir(source_dir, pkg5_dir, release_scope=False) + + # Install lines for pipeline files only + install_lines_5 = [] + pipeline_files = [ + "new_docker_build.sh", + "Dockerfile", + "Dockerfile.ubuntu26", + "jenkins-ci-rme/release-16-10-2026.sh", + "rpmbuild/make_rpm_ubuntu26.py", + "CMakeLists.txt", + "cmake/FindGeographicLib.cmake", + "cmake/FindRTI.cmake", + "cmake/FindZLIB.cmake", + ] + + for pipeline_file in pipeline_files: + src_file = os.path.join(pkg5_dir, pipeline_file) + if os.path.exists(src_file): + dest_dir = os.path.join("usr/src/pipeline", os.path.dirname(pipeline_file)) + install_lines_5.append(f"{pipeline_file} {dest_dir}/") + info(f" Added pipeline file: {pipeline_file}") + + build_single_deb_nobuild( + pkg_name=pkg5_name, + pkg_dir=pkg5_dir, + output_dir=output_dir, + version=version, + build_number=build_number, + install_lines=install_lines_5, + description="Qt6 Release Build Pipeline - Docker, Jenkins, CMake, and packaging scripts", + section="devel" + ) + + manifest_5 = create_manifest(output_dir, pkg5_name, pkg5_dir) + validate_package_contents(pkg5_name, manifest_5, + allowed_patterns=["usr/src/pipeline/*", "debian/*"], + forbidden_patterns=["*.cpp", "*.h", "tests/*"]) + # Cleanup os.chdir(source_dir) remove_directory(build_base) + info("\n========================================") info(f"DEB package build completed: {output_dir}") + info("Generated packages:") + for f in os.listdir(output_dir): + if f.endswith(".deb"): + info(f" - {f}") + info("========================================") def main(): - parser = argparse.ArgumentParser(description="Build DEB packages for Legatus LTS") + parser = argparse.ArgumentParser(description="Build DEB packages for Legatus Qt6 Release") parser.add_argument("-c", "--config", required=True, help="Config type (LTS)") parser.add_argument("--project_name", required=True, help="Project name (qatar|ahtc|guez)") parser.add_argument("--version", required=True, help="Version (Major.minor)") @@ -428,7 +886,7 @@ def main(): parser.add_argument("--source_dir", required=True, help="Source directory path") parser.add_argument("--cmake_build_flags", default="", help="CMake build flags") parser.add_argument("--build_number", default="1", help="Build number") - parser.add_argument("--release_scope", action="store_true", help="Only include release scope files (TetraIF, Z/src, Lts/Common)") + parser.add_argument("--release_scope", action="store_true", help="Only include release scope files") args = parser.parse_args() @@ -438,12 +896,11 @@ def main(): info(f"CMake flags: {args.cmake_build_flags}") info(f"Release scope: {args.release_scope}") - # Use provided source directory source_dir = args.source_dir info(f"Source directory: {source_dir}") - build_deb( + build_all_debs( source_dir=source_dir, output_dir=args.output_dir, project_name=args.project_name, @@ -452,9 +909,7 @@ def main(): cmake_flags=args.cmake_build_flags, release_scope=args.release_scope ) - - info("Build completed successfully!") if __name__ == "__main__": - main() \ No newline at end of file + main()