From 4fc47a774f37e20b9d74b24a712a14d7c2ae685e Mon Sep 17 00:00:00 2001
From: Anonymous Maarten <[EMAIL REDACTED]>
Date: Sat, 2 Nov 2024 02:01:30 +0100
Subject: [PATCH] port build-script from SDL3
[ci skip]
---
.github/workflows/release.yml | 23 +-
build-scripts/build-release.py | 1464 +++++++++++++----
build-scripts/create-release.py | 18 +-
build-scripts/release-info.json | 105 ++
mingw/pkg-support/Makefile | 11 +-
.../cmake/sdl2-config-version.cmake | 4 +-
mingw/pkg-support/cmake/sdl2-config.cmake | 4 +-
7 files changed, 1272 insertions(+), 357 deletions(-)
create mode 100644 build-scripts/release-info.json
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f6f0c56665a8d..c740c2399158a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -37,9 +37,8 @@ jobs:
shell: bash
run: |
python build-scripts/build-release.py \
- --create source \
+ --actions source \
--commit ${{ inputs.commit }} \
- --project SDL2 \
--root "${{ github.workspace }}/SDL" \
--github \
--debug
@@ -93,7 +92,7 @@ jobs:
- name: 'Set up Python'
uses: actions/setup-python@v5
with:
- python-version: '3.10'
+ python-version: '3.11'
- name: 'Fetch build-release.py'
uses: actions/checkout@v4
with:
@@ -114,9 +113,8 @@ jobs:
shell: bash
run: |
python build-scripts/build-release.py \
- --create framework \
+ --actions dmg \
--commit ${{ inputs.commit }} \
- --project SDL2 \
--root "${{ steps.tar.outputs.path }}" \
--github \
--debug
@@ -192,7 +190,7 @@ jobs:
- name: 'Set up Python'
uses: actions/setup-python@v5
with:
- python-version: '3.10'
+ python-version: '3.11'
- name: 'Fetch build-release.py'
uses: actions/checkout@v4
with:
@@ -213,9 +211,8 @@ jobs:
id: releaser
run: |
python build-scripts/build-release.py `
- --create win32 `
+ --actions msvc `
--commit ${{ inputs.commit }} `
- --project SDL2 `
--root "${{ steps.zip.outputs.path }}" `
--github `
--debug
@@ -310,7 +307,7 @@ jobs:
- name: 'Set up Python'
uses: actions/setup-python@v5
with:
- python-version: '3.10'
+ python-version: '3.11'
- name: 'Fetch build-release.py'
uses: actions/checkout@v4
with:
@@ -334,9 +331,8 @@ jobs:
id: releaser
run: |
python build-scripts/build-release.py \
- --create mingw \
+ --actions mingw \
--commit ${{ inputs.commit }} \
- --project SDL2 \
--root "${{ steps.tar.outputs.path }}" \
--github \
--debug
@@ -370,12 +366,13 @@ jobs:
mkdir -p /tmp/tardir
tar -C /tmp/tardir -v -x -f "${{ github.workspace }}/${{ needs.src.outputs.src-tar-gz }}"
echo "path=/tmp/tardir/${{ needs.src.outputs.project }}-${{ needs.src.outputs.version }}" >>$GITHUB_OUTPUT
- - name: 'Untar ${{ needs.mingw.outputs.mingw-devel-tar-gz }}'
+ - name: 'Untar and install ${{ needs.mingw.outputs.mingw-devel-tar-gz }}'
id: bin
run: |
mkdir -p /tmp/mingw-tardir
tar -C /tmp/mingw-tardir -v -x -f "${{ github.workspace }}/${{ needs.mingw.outputs.mingw-devel-tar-gz }}"
- echo "path=/tmp/mingw-tardir/${{ needs.src.outputs.project }}-${{ needs.src.outputs.version }}" >>$GITHUB_OUTPUT
+ make -C /tmp/mingw-tardir/${{ needs.src.outputs.project }}-${{ needs.src.outputs.version }} cross CROSS_PATH=/tmp/deps-mingw
+ echo "path=/tmp/deps-mingw" >>$GITHUB_OUTPUT
- name: 'CMake (configure + build) i686'
run: |
set -e
diff --git a/build-scripts/build-release.py b/build-scripts/build-release.py
index 0da88a075bcb8..5f3f7fa0d4fba 100755
--- a/build-scripts/build-release.py
+++ b/build-scripts/build-release.py
@@ -1,17 +1,28 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+
+"""
+This script is shared between SDL2, SDL3, and all satellite libraries.
+Don't specialize this script for doing project-specific modifications.
+Rather, modify release-info.json.
+"""
import argparse
import collections
+import dataclasses
+from collections.abc import Callable
import contextlib
import datetime
+import fnmatch
import glob
import io
import json
import logging
+import multiprocessing
import os
from pathlib import Path
import platform
import re
+import shlex
import shutil
import subprocess
import sys
@@ -21,21 +32,55 @@
import typing
import zipfile
-logger = logging.getLogger(__name__)
-
-VcArchDevel = collections.namedtuple("VcArchDevel", ("dll", "pdb", "imp", "main", "test"))
+logger = logging.getLogger(__name__)
GIT_HASH_FILENAME = ".git-hash"
-
-ANDROID_AVAILABLE_ABIS = [
- "armeabi-v7a",
- "arm64-v8a",
- "x86",
- "x86_64",
-]
-ANDROID_MINIMUM_API = 19
-ANDROID_TARGET_API = 29
-ANDROID_MINIMUM_NDK = 21
+REVISION_TXT = "REVISION.txt"
+
+
+def safe_isotime_to_datetime(str_isotime: str) -> datetime.datetime:
+ try:
+ return datetime.datetime.fromisoformat(str_isotime)
+ except ValueError:
+ pass
+ logger.warning("Invalid iso time: %s", str_isotime)
+ if str_isotime[-6:-5] in ("+", "-"):
+ # Commits can have isotime with invalid timezone offset (e.g. "2021-07-04T20:01:40+32:00")
+ modified_str_isotime = str_isotime[:-6] + "+00:00"
+ try:
+ return datetime.datetime.fromisoformat(modified_str_isotime)
+ except ValueError:
+ pass
+ raise ValueError(f"Invalid isotime: {str_isotime}")
+
+
+def arc_join(*parts: list[str]) -> str:
+ assert all(p[:1] != "/" and p[-1:] != "/" for p in parts), f"None of {parts} may start or end with '/'"
+ return "/".join(p for p in parts if p)
+
+
+@dataclasses.dataclass(frozen=True)
+class VsArchPlatformConfig:
+ arch: str
+ configuration: str
+ platform: str
+
+ def extra_context(self):
+ return {
+ "ARCH": self.arch,
+ "CONFIGURATION": self.configuration,
+ "PLATFORM": self.platform,
+ }
+
+
+@contextlib.contextmanager
+def chdir(path):
+ original_cwd = os.getcwd()
+ try:
+ os.chdir(path)
+ yield
+ finally:
+ os.chdir(original_cwd)
class Executer:
@@ -43,14 +88,18 @@ def __init__(self, root: Path, dry: bool=False):
self.root = root
self.dry = dry
- def run(self, cmd, stdout=False, dry_out=None, force=False):
+ def run(self, cmd, cwd=None, env=None):
+ logger.info("Executing args=%r", cmd)
sys.stdout.flush()
+ if not self.dry:
+ subprocess.check_call(cmd, cwd=cwd or self.root, env=env, text=True)
+
+ def check_output(self, cmd, cwd=None, dry_out=None, env=None, text=True):
logger.info("Executing args=%r", cmd)
- if self.dry and not force:
- if stdout:
- return subprocess.run(["echo", "-E", dry_out or ""], stdout=subprocess.PIPE if stdout else None, text=True, check=True, cwd=self.root)
- else:
- return subprocess.run(cmd, stdout=subprocess.PIPE if stdout else None, text=True, check=True, cwd=self.root)
+ sys.stdout.flush()
+ if self.dry:
+ return dry_out
+ return subprocess.check_output(cmd, cwd=cwd or self.root, env=env, text=text)
class SectionPrinter:
@@ -103,7 +152,7 @@ def find_vsdevcmd(self, year: typing.Optional[str]=None) -> typing.Optional[Path
return None
vswhere_spec.extend(["-version", f"[{version},{version+1})"])
vswhere_cmd = ["vswhere"] + vswhere_spec + ["-property", "installationPath"]
- vs_install_path = Path(self.executer.run(vswhere_cmd, stdout=True, dry_out="/tmp").stdout.strip())
+ vs_install_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp").strip())
logger.info("VS install_path = %s", vs_install_path)
assert vs_install_path.is_dir(), "VS installation path does not exist"
vsdevcmd_path = vs_install_path / "Common7/Tools/vsdevcmd.bat"
@@ -116,7 +165,7 @@ def find_vsdevcmd(self, year: typing.Optional[str]=None) -> typing.Optional[Path
def find_msbuild(self) -> typing.Optional[Path]:
vswhere_cmd = ["vswhere", "-latest", "-requires", "Microsoft.Component.MSBuild", "-find", r"MSBuild\**\Bin\MSBuild.exe"]
- msbuild_path = Path(self.executer.run(vswhere_cmd, stdout=True, dry_out="/tmp/MSBuild.exe").stdout.strip())
+ msbuild_path = Path(self.executer.check_output(vswhere_cmd, dry_out="/tmp/MSBuild.exe").strip())
logger.info("MSBuild path = %s", msbuild_path)
if self.dry:
msbuild_path.parent.mkdir(parents=True, exist_ok=True)
@@ -124,11 +173,11 @@ def find_msbuild(self) -> typing.Optional[Path]:
assert msbuild_path.is_file(), "MSBuild.exe does not exist"
return msbuild_path
- def build(self, arch: str, platform: str, configuration: str, projects: list[Path]):
+ def build(self, arch_platform: VsArchPlatformConfig, projects: list[Path]):
assert projects, "Need at least one project to build"
- vsdev_cmd_str = f"\"{self.vsdevcmd}\" -arch={arch}"
- msbuild_cmd_str = " && ".join([f"\"{self.msbuild}\" \"{project}\" /m /p:BuildInParallel=true /p:Platform={platform} /p:Configuration={configuration}" for project in projects])
+ vsdev_cmd_str = f"\"{self.vsdevcmd}\" -arch={arch_platform.arch}"
+ msbuild_cmd_str = " && ".join([f"\"{self.msbuild}\" \"{project}\" /m /p:BuildInParallel=true /p:Platform={arch_platform.platform} /p:Configuration={arch_platform.configuration}" for project in projects])
bat_contents = f"{vsdev_cmd_str} && {msbuild_cmd_str}\n"
bat_path = Path(tempfile.gettempdir()) / "cmd.bat"
with bat_path.open("w") as f:
@@ -139,35 +188,310 @@ def build(self, arch: str, platform: str, configuration: str, projects: list[Pat
self.executer.run(cmd)
-class Releaser:
- def __init__(self, project: str, commit: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str):
- self.project = project
- self.version = self.extract_sdl_version(root=root, project=project)
+class Archiver:
+ def __init__(self, zip_path: typing.Optional[Path]=None, tgz_path: typing.Optional[Path]=None, txz_path: typing.Optional[Path]=None):
+ self._zip_files = []
+ self._tar_files = []
+ self._added_files = set()
+ if zip_path:
+ self._zip_files.append(zipfile.ZipFile(zip_path, "w", compression=zipfile.ZIP_DEFLATED))
+ if tgz_path:
+ self._tar_files.append(tarfile.open(tgz_path, "w:gz"))
+ if txz_path:
+ self._tar_files.append(tarfile.open(txz_path, "w:xz"))
+
+ @property
+ def added_files(self) -> set[str]:
+ return self._added_files
+
+ def add_file_data(self, arcpath: str, data: bytes, mode: int, time: datetime.datetime):
+ for zf in self._zip_files:
+ file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
+ zip_info = zipfile.ZipInfo(filename=arcpath, date_time=file_data_time)
+ zip_info.external_attr = mode << 16
+ zip_info.compress_type = zipfile.ZIP_DEFLATED
+ zf.writestr(zip_info, data=data)
+ for tf in self._tar_files:
+ tar_info = tarfile.TarInfo(arcpath)
+ tar_info.type = tarfile.REGTYPE
+ tar_info.mode = mode
+ tar_info.size = len(data)
+ tar_info.mtime = int(time.timestamp())
+ tf.addfile(tar_info, fileobj=io.BytesIO(data))
+
+ self._added_files.add(arcpath)
+
+ def add_symlink(self, arcpath: str, target: str, time: datetime.datetime, files_for_zip):
+ logger.debug("Adding symlink (target=%r) -> %s", target, arcpath)
+ for zf in self._zip_files:
+ file_data_time = (time.year, time.month, time.day, time.hour, time.minute, time.second)
+ for f in files_for_zip:
+ zip_info = zipfile.ZipInfo(filename=f["arcpath"], date_time=file_data_time)
+ zip_info.external_attr = f["mode"] << 16
+ zip_info.compress_type = zipfile.ZIP_DEFLATED
+ zf.writestr(zip_info, data=f["data"])
+ for tf in self._tar_files:
+ tar_info = tarfile.TarInfo(arcpath)
+ tar_info.type = tarfile.SYMTYPE
+ tar_info.mode = 0o777
+ tar_info.mtime = int(time.timestamp())
+ tar_info.linkname = target
+ tf.addfile(tar_info)
+
+ self._added_files.update(f["arcpath"] for f in files_for_zip)
+
+ def add_git_hash(self, arcdir: str, commit: str, time: datetime.datetime):
+ arcpath = arc_join(arcdir, GIT_HASH_FILENAME)
+ data = f"{commit}\n".encode()
+ self.add_file_data(arcpath=arcpath, data=data, mode=0o100644, time=time)
+
+ def add_file_path(self, arcpath: str, path: Path):
+ assert path.is_file(), f"{path} should be a file"
+ logger.debug("Adding %s -> %s", path, arcpath)
+ for zf in self._zip_files:
+ zf.write(path, arcname=arcpath)
+ for tf in self._tar_files:
+ tf.add(path, arcname=arcpath)
+
+ def add_file_directory(self, arcdirpath: str, dirpath: Path):
+ assert dirpath.is_dir()
+ if arcdirpath and arcdirpath[-1:] != "/":
+ arcdirpath += "/"
+ for f in dirpath.iterdir():
+ if f.is_file():
+ arcpath = f"{arcdirpath}{f.name}"
+ logger.debug("Adding %s to %s", f, arcpath)
+ self.add_file_path(arcpath=arcpath, path=f)
+
+ def close(self):
+ # Archiver is intentionally made invalid after this function
+ del self._zip_files
+ self._zip_files = None
+ del self._tar_files
+ self._tar_files = None
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, type, value, traceback):
+ self.close()
+
+
+class NodeInArchive:
+ def __init__(self, arcpath: str, path: typing.Optional[Path]=None, data: typing.Optional[bytes]=None, mode: typing.Optional[int]=None, symtarget: typing.Optional[str]=None, time: typing.Optional[datetime.datetime]=None, directory: bool=False):
+ self.arcpath = arcpath
+ self.path = path
+ self.data = data
+ self.mode = mode
+ self.symtarget = symtarget
+ self.time = time
+ self.directory = directory
+
+ @classmethod
+ def from_fs(cls, arcpath: str, path: Path, mode: int=0o100644, time: typing.Optional[datetime.datetime]=None) -> "NodeInArchive":
+ if time is None:
+ time = datetime.datetime.fromtimestamp(os.stat(path).st_mtime)
+ return cls(arcpath=arcpath, path=path, mode=mode)
+
+ @classmethod
+ def from_data(cls, arcpath: str, data: bytes, time: datetime.datetime) -> "NodeInArchive":
+ return cls(arcpath=arcpath, data=data, time=time, mode=0o100644)
+
+ @classmethod
+ def from_text(cls, arcpath: str, text: str, time: datetime.datetime) -> "NodeInArchive":
+ return cls.from_data(arcpath=arcpath, data=text.encode(), time=time)
+
+ @classmethod
+ def from_symlink(cls, arcpath: str, symtarget: str) -> "NodeInArchive":
+ return cls(arcpath=arcpath, symtarget=symtarget)
+
+ @classmethod
+ def from_directory(cls, arcpath: str) -> "NodeInArchive":
+ return cls(arcpath=arcpath, directory=True)
+
+ def __repr__(self) -> str:
+ return f"<{type(self).__name__}:arcpath={self.arcpath},path='{str(self.path)}',len(data)={len(self.data) if self.data else 'n/a'},directory={self.directory},symtarget={self.symtarget}>"
+
+
+def configure_file(path: Path, context: dict[str, str]) -> bytes:
+ text = path.read_text()
+ return configure_text(text, context=context).encode()
+
+
+def configure_text(text: str, context: dict[str, str]) -> str:
+ original_text = text
+ for txt, repl in context.items():
+ text = text.replace(f"@<@{txt}@>@", repl)
+ success = all(thing not in text for thing in ("@<@", "@>@"))
+ if not success:
+ raise ValueError(f"Failed to configure {repr(original_text)}")
+ return text
+
+
+class ArchiveFileTree:
+ def __init__(self):
+ self._tree: dict[str, NodeInArchive] = {}
+
+ def add_file(self, file: NodeInArchive):
+ self._tree[file.arcpath] = file
+
+ def get_latest_mod_time(self) -> datetime.datetime:
+ return max(item.time for item in self._tree.values() if item.time)
+
+ def add_to_archiver(self, archive_base: str, archiver: Archiver):
+ remaining_symlinks = set()
+ added_files = dict()
+
+ def calculate_symlink_target(s: NodeInArchive) -> str:
+ dest_dir = os.path.dirname(s.arcpath)
+ if dest_dir:
+ dest_dir += "/"
+ target = dest_dir + s.symtarget
+ while True:
+ new_target, n = re.subn(r"([^/]+/+[.]{2}/)", "", target)
+ print(f"{target=} {new_target=}")
+ target = new_target
+ if not n:
+ break
+ return target
+
+ # Add files in first pass
+ for arcpath, node in self._tree.items():
+ if node.data is not None:
+ archiver.add_file_data(arcpath=arc_join(archive_base, arcpath), data=node.data, time=node.time, mode=node.mode)
+ assert node.arcpath is not None, f"{node=} has arcpath=None"
+ added_files[node.arcpath] = node
+ elif node.path is not None:
+ archiver.add_file_path(arcpath=arc_join(archive_base, arcpath), path=node.path)
+ assert node.arcpath is not None, f"{node=} has arcpath=None"
+ added_files[node.arcpath] = node
+ elif node.symtarget is not None:
+ remaining_symlinks.add(node)
+ elif node.directory:
+ pass
+ else:
+ raise ValueError(f"Invalid Archive Node: {repr(node)}")
+
+ # Resolve symlinks in second pass: zipfile does not support symlinks, so add files to zip archive
+ while True:
+ if not remaining_symlinks:
+ break
+ symlinks_this_time = set()
+ extra_added_files = {}
+ for symlink in remaining_symlinks:
+ symlink_files_for_zip = {}
+ symlink_target_path = calculate_symlink_target(symlink)
+ if symlink_target_path in added_files:
+ symlink_files_for_zip[symlink.arcpath] = added_files[symlink_target_path]
+ else:
+ symlink_target_path_slash = symlink_target_path + "/"
+ for added_file in added_files:
+ if added_file.startswith(symlink_target_path_slash):
+ path_in_symlink = symlink.arcpath + "/" + added_file.removeprefix(symlink_target_path_slash)
+ symlink_files_for_zip[path_in_symlink] = added_files[added_file]
+ if symlink_files_for_zip:
+ symlinks_this_time.add(symlink)
+ extra_added_files.update(symlink_files_for_zip)
+ files_for_zip = [{"arcpath": f"{archive_base}/{sym_path}", "data": sym_info.data, "mode": sym_info.mode} for sym_path, sym_info in symlink_files_for_zip.items()]
+ archiver.add_symlink(arcpath=f"{archive_base}/{symlink.arcpath}", target=symlink.symtarget, time=symlink.time, files_for_zip=files_for_zip)
+ # if not symlinks_this_time:
+ # logger.info("files added: %r", set(path for path in added_files.keys()))
+ assert symlinks_this_time, f"No targets found for symlinks: {remaining_symlinks}"
+ remaining_symlinks.difference_update(symlinks_this_time)
+ added_files.update(extra_added_files)
+
+ def add_directory_tree(self, arc_dir: str, path: Path, time: datetime.datetime):
+ assert path.is_dir()
+ for files_dir, _, filenames in os.walk(path):
+ files_dir_path = Path(files_dir)
+ rel_files_path = files_dir_path.relative_to(path)
+ for filename in filenames:
+ self.add_file(NodeInArchive.from_fs(arcpath=arc_join(arc_dir, str(rel_files_path), filename), path=files_dir_path / filename, time=time))
+
+ def _add_files_recursively(self, arc_dir: str, paths: list[Path], time: datetime.datetime):
+ logger.debug(f"_add_files_recursively({arc_dir=} {paths=})")
+ for path in paths:
+ arcpath = arc_join(arc_dir, path.name)
+ if path.is_file():
+ logger.debug("Adding %s as %s", path, arcpath)
+ self.add_file(NodeInArchive.from_fs(arcpath=arcpath, path=path, time=time))
+ elif path.is_dir():
+ self._add_files_recursively(arc_dir=arc_join(arc_dir, path.name), paths=list(path.iterdir()), time=time)
+ else:
+ raise ValueError(f"Unsupported file type to add recursively: {path}")
+
+ def add_file_mapping(self, arc_dir: str, file_mapping: dict[str, list[str]], file_mapping_root: Path, context: dict[str, str], time: datetime.datetime):
+ for meta_rel_destdir, meta_file_globs in file_mapping.items():
+ rel_destdir = configure_text(meta_rel_destdir, context=context)
+ assert "@" not in rel_destdir, f"archive destination should not contain an @ after configuration ({repr(meta_rel_destdir)}->{repr(rel_destdir)})"
+ for meta_file_glob in meta_file_globs:
+ file_glob = configure_text(meta_file_glob, context=context)
+ assert "@" not in rel_destdir, f"archive glob should not contain an @ after configuration ({repr(meta_file_glob)}->{repr(file_glob)})"
+ if ":" in file_glob:
+ original_path, new_filename = file_glob.rsplit(":", 1)
+ assert ":" not in original_path, f"Too many ':' in {repr(file_glob)}"
+ assert "/" not in new_filename, f"New filename cannot contain a '/' in {repr(file_glob)}"
+ path = file_mapping_root / original_path
+ arcpath = arc_join(arc_dir, rel_destdir, new_filename)
+ if path.suffix == ".in":
+ data = configure_file(path, context=context)
+ logger.debug("Adding processed %s -> %s", path, arcpath)
+ self.add_file(NodeInArchive.from_data(arcpath=arcpath, data=data, time=time))
+ else:
+ logger.debug("Adding %s -> %s", path, arcpath)
+ self.add_file(NodeInArchive.from_fs(arcpath=arcpath, path=path, time=time))
+ else:
+ relative_file_paths = glob.glob(file_glob, root_dir=file_mapping_root)
+ assert relative_file_paths, f"Glob '{file_glob}' does not match any file"
+ self._add_files_recursively(arc_dir=arc_join(arc_dir, rel_destdir), paths=[file_mapping_root / p for p in relative_file_paths], time=time)
+
+
+class SourceCollector:
+ # TreeItem = collections.namedtuple("TreeItem", ("path", "mode", "data", "symtarget", "directory", "time"))
+ def __init__(self, root: Path, commit: str, filter: typing.Optional[Callable[[str], bool]], executer: Executer):
self.root = root
self.commit = commit
- self.dist_path = dist_path
- self.section_printer = section_printer
+ self.filter = filter
self.executer = executer
- self.cmake_generator = cmake_generator
- self.artifacts: dict[str, Path] = {}
-
- @property
- def dry(self) -> bool:
- return self.executer.dry
-
- def prepare(self):
- logger.debug("Creating dist folder")
- self.dist_path.mkdir(parents=True, exist_ok=True)
+ def get_archive_file_tree(self) -> ArchiveFileTree:
+ git_archive_args = ["git", "archive", "--format=tar.gz", self.commit, "-o", "/dev/stdout"]
+ logger.info("Executing args=%r", git_archive_args)
+ contents_tgz = subprocess.check_output(git_archive_args, cwd=self.root, text=False)
+ tar_archive = tarfile.open(fileobj=io.BytesIO(contents_tgz), mode="r:gz")
+ filenames = tuple(m.name for m in tar_archive if (m.isfile() or m.issym()))
+
+ file_times = self._get_file_times(paths=filenames)
+ git_contents = ArchiveFileTree()
+ for ti in tar_archive:
+ if self.filter and not self.filter(ti.name):
+ continue
+ data = None
+ symtarget = None
+ directory = False
+ file_time = None
+ if ti.isfile():
+ contents_file = tar_archive.extractfile(ti.name)
+ data = contents_file.read()
+ file_time = file_times[ti.name]
+ elif ti.issym():
+ symtarget = ti.linkname
+ file_time = file_times[ti.name]
+ elif ti.isdir():
+ directory = True
+ else:
+ raise ValueError(f"{ti.name}: unknown type")
+ node = NodeInArchive(arcpath=ti.name, data=data, mode=ti.mode, symtarget=symtarget, time=file_time, directory=directory)
+ git_contents.add_file(node)
+ return git_contents
- TreeItem = collections.namedtuple("TreeItem", ("path", "mode", "data", "time"))
def _get_file_times(self, paths: tuple[str, ...]) -> dict[str, datetime.datetime]:
dry_out = textwrap.dedent("""\
time=2024-03-14T15:40:25-07:00
M\tCMakeLists.txt
""")
- git_log_out = self.executer.run(["git", "log", "--name-status", '--pretty=time=%cI', self.commit], stdout=True, dry_out=dry_out).stdout.splitlines(keepends=False)
+ git_log_out = self.executer.check_output(["git", "log", "--name-status", '--pretty=time=%cI', self.commit], dry_out=dry_out, cwd=self.root).splitlines(keepends=False)
current_time = None
set_paths = set(paths)
path_times: dict[str, datetime.datetime] = {}
@@ -175,98 +499,154 @@ def _get_file_times(self, paths: tuple[str, ...]) -> dict[str, datetime.datetime
if not line:
continue
if line.startswith("time="):
- current_time = datetime.datetime.fromisoformat(line.removeprefix("time="))
+ current_time = safe_isotime_to_datetime(line.removeprefix("time="))
continue
mod_type, file_paths = line.split(maxsplit=1)
assert current_time is not None
for file_path in file_paths.split("\t"):
if file_path in set_paths and file_path not in path_times:
path_times[file_path] = current_time
- assert set(path_times.keys()) == set_paths
+
+ # FIXME: find out why some files are not shown in "git log"
+ # assert set(path_times.keys()) == set_paths
+ if set(path_times.keys()) != set_paths:
+ found_times = set(path_times.keys())
+ paths_without_times = set_paths.difference(found_times)
+ logger.warning("No times found for these paths: %s", paths_without_times)
+ max_time = max(time for time in path_times.values())
+ for path in paths_without_times:
+ path_times[path] = max_time
+
return path_times
- @staticmethod
- def _path_filter(path: str):
+
+class Releaser:
+ def __init__(self, release_info: dict, commit: str, revision: str, root: Path, dist_path: Path, section_printer: SectionPrinter, executer: Executer, cmake_generator: str, deps_path: Path, overwrite: bool, github: bool, fast: bool):
+ self.release_info = release_info
+ self.project = release_info["name"]
+ self.version = self.extract_sdl_version(root=root, release_info=release_info)
+ self.root = root
+ self.commit = commit
+ self.revision = revision
+ self.dist_path = dist_path
+ self.section_printer = section_printer
+ self.executer = executer
+ self.cmake_generator = cmake_generator
+ self.cpu_count = multiprocessing.cpu_count()
+ self.deps_path = deps_path
+ self.overwrite = overwrite
+ self.github = github
+ self.fast = fast
+ self.arc_time = datetime.datetime.now()
+
+ self.artifacts: dict[str, Path] = {}
+
+ def get_context(self, extra_context: typing.Optional[dict[str, str]]=None) -> dict[str, str]:
+ ctx = {
+ "PROJECT_NAME": self.project,
+ "PROJECT_VERSION": self.version,
+ "PROJECT_COMMIT": self.commit,
+ "PROJECT_REVISION": self.revision,
+ }
+ if extra_context:
+ ctx.update(extra_context)
+ return ctx
+
+ @property
+ def dry(self) -> bool:
+ return self.executer.dry
+
+ def prepare(self):
+ logger.debug("Creating dist folder")
+ self.dist_path.mkdir(parents=True, exist_ok=True)
+
+ @classmethod
+ def _path_filter(cls, path: str) -> bool:
+ if ".gitmodules" in path:
+ return True
if path.startswith(".git"):
return False
return True
- def _get_git_contents(self) -> dict[str, TreeItem]:
- contents_tgz = subprocess.check_output(["git", "archive", "--format=tar.gz", self.commit, "-o", "/dev/stdout"], text=False)
- contents = tarfile.open(fileobj=io.BytesIO(contents_tgz), mode="r:gz")
- filenames = tuple(m.name for m in contents if m.isfile())
- assert "src/SDL.c" in filenames
- assert "include/SDL.h" in filenames
- file_times = self._get_file_times(filenames)
- git_contents = {}
- for ti in contents:
- if not ti.isfile():
- continue
- if not self._path_filter(ti.name):
- continue
- contents_file = contents.extractfile(ti.name)
- assert contents_file, f"{ti.name} is not a file"
- git_contents[ti.name] = self.TreeItem(path=ti.name, mode=ti.mode, data=contents_file.read(), time=file_times[ti.name])
- return git_contents
+ @classmethod
+ def _external_repo_path_filter(cls, path: str) -> bool:
+ if not cls._path_filter(path):
+ return False
+ if path.startswith("test/") or path.startswith("tests/"):
+ return False
+ return True
def create_source_archives(self) -> None:
+ source_collector = SourceCollector(root=self.root, commit=self.commit, executer=self.executer, filter=self._path_filter)
+ print(f"Collecting sources of {self.project}...")
+ archive_tree = source_collector.get_archive_file_tree()
+ latest_mod_time = archive_tree.get_latest_mod_time()
+ archive_tree.add_file(N
(Patch may be truncated, please check the link at the top of this post.)