From 0e03867419d9aa1fdef9e119a582ff37eb89dcaa Mon Sep 17 00:00:00 2001 From: Gargi Gupta Date: Mon, 20 Jul 2026 04:39:47 +0530 Subject: [PATCH 1/3] fix: honor gitignore when packaging context --- kinetic/utils/packager.py | 78 +++++++++++++++++++- kinetic/utils/packager_test.py | 130 +++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 4 deletions(-) diff --git a/kinetic/utils/packager.py b/kinetic/utils/packager.py index eadc00d5..ea4651d2 100644 --- a/kinetic/utils/packager.py +++ b/kinetic/utils/packager.py @@ -6,6 +6,7 @@ """ import os +import subprocess import zipfile from collections.abc import Callable from typing import Any @@ -18,13 +19,77 @@ PositionPath = tuple[str | int, ...] +def _list_git_files(base_dir: str) -> list[str] | None: + """List tracked and non-ignored untracked files under ``base_dir``.""" + try: + result = subprocess.run( + [ + "git", + "-C", + base_dir, + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + ".", + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except (FileNotFoundError, subprocess.CalledProcessError): + return None + + return [os.fsdecode(path) for path in result.stdout.split(b"\0") if path] + + +def _path_is_excluded(path: str, exclude_paths: set[str]) -> bool: + normalized_path = os.path.normpath(path) + return any( + normalized_path == excluded or normalized_path.startswith(excluded + os.sep) + for excluded in exclude_paths + ) + + +def _write_git_files( + zipf: zipfile.ZipFile, + base_dir: str, + git_files: list[str], + exclude_paths: set[str], + archive_prefix: str = "", +) -> None: + for relative_path in git_files: + file_path = os.path.join(base_dir, relative_path) + if _path_is_excluded(file_path, exclude_paths) or not os.path.lexists( + file_path + ): + continue + + archive_name = os.path.join(archive_prefix, relative_path) + if os.path.isdir(file_path) and not os.path.islink(file_path): + nested_files = _list_git_files(file_path) + if nested_files is not None: + _write_git_files( + zipf, + file_path, + nested_files, + exclude_paths, + archive_prefix=archive_name, + ) + continue + zipf.write(file_path, archive_name) + + def zip_working_dir( base_dir: str, output_path: str, exclude_paths: set[str] | None = None ) -> None: - """Zip a directory into a ZIP archive, excluding common non-source files. + """Zip source files from a working directory. - Excludes ``.git``, ``__pycache__``, and any paths in *exclude_paths* - (which may be files or directories). + In a Git worktree, includes tracked files and non-ignored untracked files. + Otherwise, walks the directory and excludes ``.git`` and ``__pycache__``. + Paths in *exclude_paths* are always excluded. Args: base_dir: Root directory to zip. @@ -33,8 +98,13 @@ def zip_working_dir( """ exclude_paths = exclude_paths or set() normalized_excludes = {os.path.normpath(p) for p in exclude_paths} + git_files = _list_git_files(base_dir) with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: + if git_files is not None: + _write_git_files(zipf, base_dir, git_files, normalized_excludes) + return + for root, dirs, files in os.walk(base_dir): # Exclude .git, __pycache__, and Data-referenced directories dirs[:] = [ @@ -46,7 +116,7 @@ def zip_working_dir( for file in files: file_path = os.path.join(root, file) - if os.path.normpath(file_path) in normalized_excludes: + if _path_is_excluded(file_path, normalized_excludes): continue archive_name = os.path.relpath(file_path, base_dir) zipf.write(file_path, archive_name) diff --git a/kinetic/utils/packager_test.py b/kinetic/utils/packager_test.py index f86b6222..abe1361b 100644 --- a/kinetic/utils/packager_test.py +++ b/kinetic/utils/packager_test.py @@ -2,6 +2,8 @@ import os import pathlib +import shutil +import subprocess import tempfile import zipfile @@ -33,6 +35,14 @@ def _zip_and_list(self, src, tmp_path, exclude_paths=None): with zipfile.ZipFile(str(out)) as zf: return set(zf.namelist()) + def _init_git_repo(self, path): + if shutil.which("git") is None: + self.skipTest("Git is required for this test.") + subprocess.run( + ["git", "init", "--quiet", str(path)], + check=True, + ) + def test_contains_all_files(self): tmp_path = _make_temp_path(self) src = tmp_path / "src" @@ -127,6 +137,126 @@ def test_exclude_multiple_paths(self): names = self._zip_and_list(src, tmp_path, exclude_paths={str(d1), str(d2)}) self.assertEqual(names, {"main.py"}) + def test_git_repository_respects_gitignore(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + self._init_git_repo(src) + (src / ".gitignore").write_text(".venv/\n*.log\n") + (src / "main.py").write_text("code") + (src / "debug.log").write_text("logs") + venv = src / ".venv" + venv.mkdir() + (venv / "python").write_text("binary") + + names = self._zip_and_list(src, tmp_path) + + self.assertEqual(names, {".gitignore", "main.py"}) + + def test_git_repository_includes_tracked_ignored_file(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + self._init_git_repo(src) + (src / ".gitignore").write_text("tracked.txt\n") + tracked = src / "tracked.txt" + tracked.write_text("tracked") + subprocess.run( + ["git", "-C", str(src), "add", "-f", "tracked.txt"], + check=True, + ) + + names = self._zip_and_list(src, tmp_path) + + self.assertIn("tracked.txt", names) + + def test_git_repository_preserves_explicit_exclusions(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + self._init_git_repo(src) + data_dir = src / "data" + data_dir.mkdir() + (data_dir / "large.bin").write_text("data") + (src / "main.py").write_text("code") + + names = self._zip_and_list(src, tmp_path, exclude_paths={str(data_dir)}) + + self.assertEqual(names, {"main.py"}) + + def test_git_repository_supports_subdirectory_working_dir(self): + tmp_path = _make_temp_path(self) + repo = tmp_path / "repo" + src = repo / "package" + src.mkdir(parents=True) + self._init_git_repo(repo) + (repo / ".gitignore").write_text("package/generated/\n") + (src / "main.py").write_text("code") + generated = src / "generated" + generated.mkdir() + (generated / "weights.bin").write_text("weights") + + names = self._zip_and_list(src, tmp_path) + + self.assertEqual(names, {"main.py"}) + + def test_git_repository_skips_deleted_tracked_file(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + self._init_git_repo(src) + deleted = src / "deleted.py" + deleted.write_text("old code") + subprocess.run( + ["git", "-C", str(src), "add", "deleted.py"], + check=True, + ) + deleted.unlink() + (src / "main.py").write_text("code") + + names = self._zip_and_list(src, tmp_path) + + self.assertEqual(names, {"main.py"}) + + def test_git_repository_includes_submodule_files(self): + tmp_path = _make_temp_path(self) + repo = tmp_path / "repo" + submodule = repo / "vendor" + submodule.mkdir(parents=True) + self._init_git_repo(repo) + self._init_git_repo(submodule) + (submodule / "module.py").write_text("code") + subprocess.run( + ["git", "-C", str(submodule), "add", "module.py"], + check=True, + ) + subprocess.run( + [ + "git", + "-C", + str(submodule), + "-c", + "user.name=Kinetic Tests", + "-c", + "user.email=kinetic-tests@example.com", + "commit", + "--quiet", + "-m", + "Initial commit", + ], + check=True, + ) + subprocess.run( + ["git", "-C", str(repo), "add", "vendor"], + check=True, + stderr=subprocess.DEVNULL, + ) + (repo / "main.py").write_text("code") + + names = self._zip_and_list(repo, tmp_path) + + self.assertEqual(names, {"main.py", os.path.join("vendor", "module.py")}) + class TestSavePayload(absltest.TestCase): def _save_and_load( From 7d58b1b2f3a7fb4a47745365259206160e5ff841 Mon Sep 17 00:00:00 2001 From: Gargi Gupta Date: Mon, 20 Jul 2026 05:09:03 +0530 Subject: [PATCH 2/3] fix: harden git-aware packaging --- kinetic/utils/packager.py | 8 +++++-- kinetic/utils/packager_test.py | 38 ++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/kinetic/utils/packager.py b/kinetic/utils/packager.py index ea4651d2..3e30ab65 100644 --- a/kinetic/utils/packager.py +++ b/kinetic/utils/packager.py @@ -6,6 +6,7 @@ """ import os +import posixpath import subprocess import zipfile from collections.abc import Callable @@ -39,13 +40,15 @@ def _list_git_files(base_dir: str) -> list[str] | None: stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, ) - except (FileNotFoundError, subprocess.CalledProcessError): + except (OSError, subprocess.CalledProcessError): return None return [os.fsdecode(path) for path in result.stdout.split(b"\0") if path] def _path_is_excluded(path: str, exclude_paths: set[str]) -> bool: + if not exclude_paths: + return False normalized_path = os.path.normpath(path) return any( normalized_path == excluded or normalized_path.startswith(excluded + os.sep) @@ -67,7 +70,7 @@ def _write_git_files( ): continue - archive_name = os.path.join(archive_prefix, relative_path) + archive_name = posixpath.join(archive_prefix, relative_path) if os.path.isdir(file_path) and not os.path.islink(file_path): nested_files = _list_git_files(file_path) if nested_files is not None: @@ -98,6 +101,7 @@ def zip_working_dir( """ exclude_paths = exclude_paths or set() normalized_excludes = {os.path.normpath(p) for p in exclude_paths} + base_dir = os.path.abspath(base_dir) git_files = _list_git_files(base_dir) with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: diff --git a/kinetic/utils/packager_test.py b/kinetic/utils/packager_test.py index abe1361b..e5172dd9 100644 --- a/kinetic/utils/packager_test.py +++ b/kinetic/utils/packager_test.py @@ -6,6 +6,7 @@ import subprocess import tempfile import zipfile +from unittest import mock import cloudpickle import numpy as np @@ -88,7 +89,7 @@ def test_preserves_nested_structure(self): names = self._zip_and_list(src, tmp_path) self.assertIn("top.py", names) - self.assertIn(os.path.join("pkg", "sub", "deep.py"), names) + self.assertIn("pkg/sub/deep.py", names) def test_empty_directory(self): tmp_path = _make_temp_path(self) @@ -184,6 +185,39 @@ def test_git_repository_preserves_explicit_exclusions(self): self.assertEqual(names, {"main.py"}) + def test_git_repository_preserves_exclusions_with_relative_base_dir(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + self._init_git_repo(src) + data_dir = src / "data" + data_dir.mkdir() + (data_dir / "large.bin").write_text("data") + (src / "main.py").write_text("code") + previous_cwd = os.getcwd() + os.chdir(tmp_path) + self.addCleanup(os.chdir, previous_cwd) + excluded_data_dir = os.path.abspath("src/data") + + names = self._zip_and_list( + pathlib.Path("src"), tmp_path, exclude_paths={excluded_data_dir} + ) + + self.assertEqual(names, {"main.py"}) + + def test_falls_back_to_directory_walk_on_git_os_error(self): + tmp_path = _make_temp_path(self) + src = tmp_path / "src" + src.mkdir() + (src / "main.py").write_text("code") + + with mock.patch( + "kinetic.utils.packager.subprocess.run", side_effect=PermissionError + ): + names = self._zip_and_list(src, tmp_path) + + self.assertEqual(names, {"main.py"}) + def test_git_repository_supports_subdirectory_working_dir(self): tmp_path = _make_temp_path(self) repo = tmp_path / "repo" @@ -255,7 +289,7 @@ def test_git_repository_includes_submodule_files(self): names = self._zip_and_list(repo, tmp_path) - self.assertEqual(names, {"main.py", os.path.join("vendor", "module.py")}) + self.assertEqual(names, {"main.py", "vendor/module.py"}) class TestSavePayload(absltest.TestCase): From 96bdd18d46a7df77b0f9eaefaad98d2c20e4b11b Mon Sep 17 00:00:00 2001 From: Gargi Gupta Date: Thu, 6 Aug 2026 09:03:38 +0530 Subject: [PATCH 3/3] Add plan_json parameter back to zip_working_dir for PR #284 compatibility --- kinetic/utils/packager.py | 44 +++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/kinetic/utils/packager.py b/kinetic/utils/packager.py index 3e30ab65..50065a25 100644 --- a/kinetic/utils/packager.py +++ b/kinetic/utils/packager.py @@ -5,6 +5,7 @@ arbitrarily nested arg structures. """ +import json import os import posixpath import subprocess @@ -86,7 +87,10 @@ def _write_git_files( def zip_working_dir( - base_dir: str, output_path: str, exclude_paths: set[str] | None = None + base_dir: str, + output_path: str, + exclude_paths: set[str] | None = None, + plan_json: dict[str, Any] | None = None, ) -> None: """Zip source files from a working directory. @@ -98,6 +102,8 @@ def zip_working_dir( base_dir: Root directory to zip. output_path: Destination path for the ZIP file. exclude_paths: Absolute paths to skip during archiving. + plan_json: Optional packaging plan, written into the archive at the + reserved path ``.kinetic/plan.json`` for the remote runner. """ exclude_paths = exclude_paths or set() normalized_excludes = {os.path.normpath(p) for p in exclude_paths} @@ -107,23 +113,25 @@ def zip_working_dir( with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zipf: if git_files is not None: _write_git_files(zipf, base_dir, git_files, normalized_excludes) - return - - for root, dirs, files in os.walk(base_dir): - # Exclude .git, __pycache__, and Data-referenced directories - dirs[:] = [ - d - for d in dirs - if d not in [".git", "__pycache__"] - and os.path.normpath(os.path.join(root, d)) not in normalized_excludes - ] - - for file in files: - file_path = os.path.join(root, file) - if _path_is_excluded(file_path, normalized_excludes): - continue - archive_name = os.path.relpath(file_path, base_dir) - zipf.write(file_path, archive_name) + else: + for root, dirs, files in os.walk(base_dir): + # Exclude .git, __pycache__, and Data-referenced directories + dirs[:] = [ + d + for d in dirs + if d not in [".git", "__pycache__"] + and os.path.normpath(os.path.join(root, d)) not in normalized_excludes + ] + + for file in files: + file_path = os.path.join(root, file) + if _path_is_excluded(file_path, normalized_excludes): + continue + archive_name = os.path.relpath(file_path, base_dir) + zipf.write(file_path, archive_name) + + if plan_json: + zipf.writestr(".kinetic/plan.json", json.dumps(plan_json)) def save_payload(