Honor .gitignore when packaging job context

kinetic/utils/packager.py

@@ -6,6 +6,7 @@
 """
 
 import os
+import subprocess
 import zipfile
 from collections.abc import Callable
 from typing import Any
@@ -18,13 +19,77 @@
 PositionPath = tuple[str | int, ...]
 
 
+def _list_git_files(base_dir: str) -> list[str] | None:
+  """List tracked and non-ignored untracked files under ``base_dir``."""
+  try:
+    result = subprocess.run(
+      [
+        "git",
+        "-C",
+        base_dir,
+        "ls-files",
+        "--cached",
+        "--others",
+        "--exclude-standard",
+        "-z",
+        "--",
+        ".",
+      ],
+      check=True,
+      stdout=subprocess.PIPE,
+      stderr=subprocess.DEVNULL,
+    )
+  except (FileNotFoundError, subprocess.CalledProcessError):
+    return None
+
+  return [os.fsdecode(path) for path in result.stdout.split(b"\0") if path]
+
+
+def _path_is_excluded(path: str, exclude_paths: set[str]) -> bool:
+  normalized_path = os.path.normpath(path)
+  return any(
+    normalized_path == excluded or normalized_path.startswith(excluded + os.sep)
+    for excluded in exclude_paths
+  )
+
+
+def _write_git_files(
+  zipf: zipfile.ZipFile,
+  base_dir: str,
+  git_files: list[str],
+  exclude_paths: set[str],
+  archive_prefix: str = "",
+) -> None:
+  for relative_path in git_files:
+    file_path = os.path.join(base_dir, relative_path)
+    if _path_is_excluded(file_path, exclude_paths) or not os.path.lexists(
+      file_path
+    ):
+      continue
+
+    archive_name = os.path.join(archive_prefix, relative_path)
+    if os.path.isdir(file_path) and not os.path.islink(file_path):
+      nested_files = _list_git_files(file_path)
+      if nested_files is not None:
+        _write_git_files(
+          zipf,
+          file_path,
+          nested_files,
+          exclude_paths,
+          archive_prefix=archive_name,
+        )
+      continue
+    zipf.write(file_path, archive_name)
+
+
 def zip_working_dir(
   base_dir: str, output_path: str, exclude_paths: set[str] | None = None
 ) -> None:
-  """Zip a directory into a ZIP archive, excluding common non-source files.
+  """Zip source files from a working directory.
 
-  Excludes ``.git``, ``__pycache__``, and any paths in *exclude_paths*
-  (which may be files or directories).
+  In a Git worktree, includes tracked files and non-ignored untracked files.
+  Otherwise, walks the directory and excludes ``.git`` and ``__pycache__``.
+  Paths in *exclude_paths* are always excluded.
 
   Args:
       base_dir: Root directory to zip.
@@ -33,8 +98,13 @@ def zip_working_dir(
   """
   exclude_paths = exclude_paths or set()
   normalized_excludes = {os.path.normpath(p) for p in exclude_paths}
+  git_files = _list_git_files(base_dir)
 
   with zipfile.ZipFile(output_path, "w", zipfile.ZIP_DEFLATED) as zipf:
+    if git_files is not None:
+      _write_git_files(zipf, base_dir, git_files, normalized_excludes)
+      return
+
     for root, dirs, files in os.walk(base_dir):
       # Exclude .git, __pycache__, and Data-referenced directories
       dirs[:] = [
@@ -46,7 +116,7 @@ def zip_working_dir(
 
       for file in files:
         file_path = os.path.join(root, file)
-        if os.path.normpath(file_path) in normalized_excludes:
+        if _path_is_excluded(file_path, normalized_excludes):
           continue
         archive_name = os.path.relpath(file_path, base_dir)
         zipf.write(file_path, archive_name)

kinetic/utils/packager_test.py

@@ -2,6 +2,8 @@
 
 import os
 import pathlib
+import shutil
+import subprocess
 import tempfile
 import zipfile
 
@@ -33,6 +35,14 @@ def _zip_and_list(self, src, tmp_path, exclude_paths=None):
     with zipfile.ZipFile(str(out)) as zf:
       return set(zf.namelist())
 
+  def _init_git_repo(self, path):
+    if shutil.which("git") is None:
+      self.skipTest("Git is required for this test.")
+    subprocess.run(
+      ["git", "init", "--quiet", str(path)],
+      check=True,
+    )
+
   def test_contains_all_files(self):
     tmp_path = _make_temp_path(self)
     src = tmp_path / "src"
@@ -127,6 +137,126 @@ def test_exclude_multiple_paths(self):
     names = self._zip_and_list(src, tmp_path, exclude_paths={str(d1), str(d2)})
     self.assertEqual(names, {"main.py"})
 
+  def test_git_repository_respects_gitignore(self):
+    tmp_path = _make_temp_path(self)
+    src = tmp_path / "src"
+    src.mkdir()
+    self._init_git_repo(src)
+    (src / ".gitignore").write_text(".venv/\n*.log\n")
+    (src / "main.py").write_text("code")
+    (src / "debug.log").write_text("logs")
+    venv = src / ".venv"
+    venv.mkdir()
+    (venv / "python").write_text("binary")
+
+    names = self._zip_and_list(src, tmp_path)
+
+    self.assertEqual(names, {".gitignore", "main.py"})
+
+  def test_git_repository_includes_tracked_ignored_file(self):
+    tmp_path = _make_temp_path(self)
+    src = tmp_path / "src"
+    src.mkdir()
+    self._init_git_repo(src)
+    (src / ".gitignore").write_text("tracked.txt\n")
+    tracked = src / "tracked.txt"
+    tracked.write_text("tracked")
+    subprocess.run(
+      ["git", "-C", str(src), "add", "-f", "tracked.txt"],
+      check=True,
+    )
+
+    names = self._zip_and_list(src, tmp_path)
+
+    self.assertIn("tracked.txt", names)
+
+  def test_git_repository_preserves_explicit_exclusions(self):
+    tmp_path = _make_temp_path(self)
+    src = tmp_path / "src"
+    src.mkdir()
+    self._init_git_repo(src)
+    data_dir = src / "data"
+    data_dir.mkdir()
+    (data_dir / "large.bin").write_text("data")
+    (src / "main.py").write_text("code")
+
+    names = self._zip_and_list(src, tmp_path, exclude_paths={str(data_dir)})
+
+    self.assertEqual(names, {"main.py"})
+
+  def test_git_repository_supports_subdirectory_working_dir(self):
+    tmp_path = _make_temp_path(self)
+    repo = tmp_path / "repo"
+    src = repo / "package"
+    src.mkdir(parents=True)
+    self._init_git_repo(repo)
+    (repo / ".gitignore").write_text("package/generated/\n")
+    (src / "main.py").write_text("code")
+    generated = src / "generated"
+    generated.mkdir()
+    (generated / "weights.bin").write_text("weights")
+
+    names = self._zip_and_list(src, tmp_path)
+
+    self.assertEqual(names, {"main.py"})
+
+  def test_git_repository_skips_deleted_tracked_file(self):
+    tmp_path = _make_temp_path(self)
+    src = tmp_path / "src"
+    src.mkdir()
+    self._init_git_repo(src)
+    deleted = src / "deleted.py"
+    deleted.write_text("old code")
+    subprocess.run(
+      ["git", "-C", str(src), "add", "deleted.py"],
+      check=True,
+    )
+    deleted.unlink()
+    (src / "main.py").write_text("code")
+
+    names = self._zip_and_list(src, tmp_path)
+
+    self.assertEqual(names, {"main.py"})
+
+  def test_git_repository_includes_submodule_files(self):
+    tmp_path = _make_temp_path(self)
+    repo = tmp_path / "repo"
+    submodule = repo / "vendor"
+    submodule.mkdir(parents=True)
+    self._init_git_repo(repo)
+    self._init_git_repo(submodule)
+    (submodule / "module.py").write_text("code")
+    subprocess.run(
+      ["git", "-C", str(submodule), "add", "module.py"],
+      check=True,
+    )
+    subprocess.run(
+      [
+        "git",
+        "-C",
+        str(submodule),
+        "-c",
+        "user.name=Kinetic Tests",
+        "-c",
+        "user.email=kinetic-tests@example.com",
+        "commit",
+        "--quiet",
+        "-m",
+        "Initial commit",
+      ],
+      check=True,
+    )
+    subprocess.run(
+      ["git", "-C", str(repo), "add", "vendor"],
+      check=True,
+      stderr=subprocess.DEVNULL,
+    )
+    (repo / "main.py").write_text("code")
+
+    names = self._zip_and_list(repo, tmp_path)
+
+    self.assertEqual(names, {"main.py", os.path.join("vendor", "module.py")})
+
 
 class TestSavePayload(absltest.TestCase):
   def _save_and_load(