++ stand cluster and slicer

ci/build-slice.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+import argparse
+import io
+import subprocess
+import sys
+import tarfile
+from pathlib import Path, PurePosixPath
+
+import yaml
+
+FLUX_KUSTOMIZATION = "kustomize.toolkit.fluxcd.io"
+KUSTOMIZATION_FILES = ("kustomization.yaml", "kustomization.yml", "Kustomization")
+
+
+class SliceError(Exception):
+    pass
+
+
+def is_remote(ref: str) -> bool:
+    return "://" in ref or ref.startswith(("git@", "github.com/", "gitlab.com/"))
+
+
+def kustomization_file(directory: Path) -> Path | None:
+    for name in KUSTOMIZATION_FILES:
+        candidate = directory / name
+        if candidate.is_file():
+            return candidate
+    return None
+
+
+def local_refs(doc: dict) -> list[str]:
+    refs = []
+    refs += doc.get("resources") or []
+    refs += doc.get("components") or []
+    refs += doc.get("patchesStrategicMerge") or []
+    refs += [p["path"] for p in doc.get("patches") or [] if isinstance(p, dict) and "path" in p]
+    for key in ("configMapGenerator", "secretGenerator"):
+        for gen in doc.get(key) or []:
+            for entry in (gen.get("files") or []) + (gen.get("envs") or []):
+                refs.append(entry.split("=", 1)[-1])
+            if gen.get("env"):
+                refs.append(gen["env"])
+    return [r for r in refs if isinstance(r, str)]
+
+
+def flux_paths(path: Path) -> list[str]:
+    try:
+        docs = list(yaml.safe_load_all(path.read_text()))
+    except yaml.YAMLError:
+        return []
+    return [
+        d["spec"]["path"]
+        for d in docs
+        if isinstance(d, dict)
+        and str(d.get("apiVersion", "")).startswith(FLUX_KUSTOMIZATION)
+        and d.get("kind") == "Kustomization"
+        and d.get("spec", {}).get("path")
+    ]
+
+
+def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]:
+    repo = repo.resolve()
+    files: set[PurePosixPath] = set()
+    entries: list[str] = []
+    seen_dirs: set[Path] = set()
+
+    def rel(p: Path) -> PurePosixPath:
+        try:
+            return PurePosixPath(p.resolve().relative_to(repo).as_posix())
+        except ValueError:
+            raise SliceError(f"ссылка за пределы репозитория: {p}")
+
+    def add_file(p: Path):
+        r = rel(p)
+        if r in files:
+            return
+        files.add(r)
+        if p.suffix in (".yaml", ".yml"):
+            for fp in flux_paths(p):
+                visit_dir(repo / fp, entry=True)
+
+    def visit_dir(d: Path, entry: bool = False):
+        d = d.resolve()
+        if d in seen_dirs:
+            return
+        seen_dirs.add(d)
+        kfile = kustomization_file(d)
+        if kfile is None:
+            raise SliceError(f"нет kustomization.yaml в {rel(d)}")
+        if entry:
+            entries.append(str(rel(d)))
+        add_file(kfile)
+        doc = yaml.safe_load(kfile.read_text()) or {}
+        for ref in local_refs(doc):
+            if is_remote(ref):
+                raise SliceError(f"удалённая ссылка {ref} в {rel(kfile)}")
+            target = (d / ref)
+            if target.is_dir():
+                visit_dir(target)
+            elif target.is_file():
+                add_file(target)
+            else:
+                raise SliceError(f"ссылка {ref} в {rel(kfile)} никуда не ведёт")
+
+    visit_dir(repo / start, entry=True)
+    return files, sorted(set(entries))
+
+
+def write_tar(repo: Path, files: set[PurePosixPath], out: Path):
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w", format=tarfile.PAX_FORMAT) as tar:
+        for f in sorted(files):
+            data = (repo / f).read_bytes()
+            info = tarfile.TarInfo(str(f))
+            info.size = len(data)
+            info.mode = 0o644
+            info.mtime = 0
+            info.uid = info.gid = 0
+            info.uname = info.gname = ""
+            tar.addfile(info, io.BytesIO(data))
+    out.write_bytes(buf.getvalue())
+
+
+def kustomize_build(path: Path) -> bytes:
+    p = subprocess.run(["kubectl", "kustomize", str(path)], capture_output=True)
+    if p.returncode != 0:
+        raise SliceError(f"kustomize build {path}: {p.stderr.decode().strip()}")
+    return p.stdout
+
+
+def verify(repo: Path, out: Path, entries: list[str]):
+    import tempfile
+    with tempfile.TemporaryDirectory() as tmp:
+        with tarfile.open(out) as tar:
+            tar.extractall(tmp, filter="data")
+        for e in entries:
+            if kustomize_build(repo / e) != kustomize_build(Path(tmp) / e):
+                raise SliceError(f"build {e} в срезе отличается от полного репозитория")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("contour")
+    ap.add_argument("--repo", default=".")
+    ap.add_argument("--out", required=True)
+    ap.add_argument("--no-verify", action="store_true")
+    args = ap.parse_args()
+    repo = Path(args.repo).resolve()
+    out = Path(args.out)
+    try:
+        files, entries = collect(repo, f"clusters/{args.contour}")
+        write_tar(repo, files, out)
+        if not args.no_verify:
+            verify(repo, out, entries)
+    except SliceError as e:
+        print(f"срез {args.contour}: {e}", file=sys.stderr)
+        return 1
+    print(f"срез {args.contour}: {len(files)} файлов, слоёв {len(entries)}: {', '.join(entries)}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

ci/tests/test_build_slice.py

@@ -0,0 +1,70 @@
+import importlib.util
+import tarfile
+from pathlib import Path
+
+import pytest
+
+spec = importlib.util.spec_from_file_location("build_slice", Path(__file__).parents[1] / "build-slice.py")
+bs = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(bs)
+
+
+def put(root: Path, rel: str, text: str):
+    p = root / rel
+    p.parent.mkdir(parents=True, exist_ok=True)
+    p.write_text(text)
+
+
+@pytest.fixture
+def repo(tmp_path):
+    put(tmp_path, "clusters/a/kustomization.yaml", "resources:\n  - ./flux-system\n  - ./layers.yaml\n")
+    put(tmp_path, "clusters/a/flux-system/kustomization.yaml", "resources:\n  - gotk-sync.yaml\n")
+    put(tmp_path, "clusters/a/flux-system/gotk-sync.yaml",
+        "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nspec:\n  path: ./clusters/a\n")
+    put(tmp_path, "clusters/a/layers.yaml",
+        "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nspec:\n  path: ./layers/a/00-vault\n")
+    put(tmp_path, "layers/a/00-vault/kustomization.yaml", "resources:\n  - ../../../infrastructure/vault/contour-template\n")
+    put(tmp_path, "infrastructure/vault/contour-template/kustomization.yaml",
+        "resources:\n  - ../base\npatches:\n  - path: vault.yaml\n")
+    put(tmp_path, "infrastructure/vault/contour-template/vault.yaml", "kind: HelmRelease\n")
+    put(tmp_path, "infrastructure/vault/base/kustomization.yaml", "resources:\n  - helmrelease.yaml\n")
+    put(tmp_path, "infrastructure/vault/base/helmrelease.yaml", "kind: HelmRelease\n")
+    put(tmp_path, "infrastructure/vault/base/unused.yaml", "kind: ConfigMap\n")
+    put(tmp_path, "infrastructure/vault/uralkal/kustomization.yaml", "resources:\n  - ../base\n")
+    put(tmp_path, "clusters/b/kustomization.yaml", "resources:\n  - ../../infrastructure/vault/uralkal\n")
+    return tmp_path
+
+
+def test_slice_follows_flux_paths_and_skips_foreign(repo):
+    files, entries = bs.collect(repo, "clusters/a")
+    names = {str(f) for f in files}
+    assert "infrastructure/vault/base/helmrelease.yaml" in names
+    assert "infrastructure/vault/contour-template/vault.yaml" in names
+    assert "layers/a/00-vault/kustomization.yaml" in names
+    assert not any("uralkal" in n or n.startswith("clusters/b") for n in names)
+    assert "infrastructure/vault/base/unused.yaml" not in names
+    assert entries == ["clusters/a", "layers/a/00-vault"]
+
+
+def test_remote_reference_stops_build(repo):
+    put(repo, "clusters/a/kustomization.yaml", "resources:\n  - https://github.com/x/y//z\n")
+    with pytest.raises(bs.SliceError, match="удалённая ссылка"):
+        bs.collect(repo, "clusters/a")
+
+
+def test_dangling_reference_stops_build(repo):
+    put(repo, "clusters/a/kustomization.yaml", "resources:\n  - ./missing.yaml\n")
+    with pytest.raises(bs.SliceError, match="никуда не ведёт"):
+        bs.collect(repo, "clusters/a")
+
+
+def test_tar_is_byte_identical_across_runs(repo, tmp_path_factory):
+    files, _ = bs.collect(repo, "clusters/a")
+    out1 = tmp_path_factory.mktemp("o") / "s.tar"
+    out2 = tmp_path_factory.mktemp("o") / "s.tar"
+    bs.write_tar(repo, files, out1)
+    (repo / "clusters/a/layers.yaml").touch()
+    bs.write_tar(repo, files, out2)
+    assert out1.read_bytes() == out2.read_bytes()
+    with tarfile.open(out1) as t:
+        assert all(m.mtime == 0 for m in t.getmembers())

clusters/stand/flux-system/gotk-sync.yaml

@@ -0,0 +1,25 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: GitRepository
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 1m0s
+  ref:
+    branch: main
+  url: http://gitea.gitea.svc:3000/infra/iac.git
+  secretRef:
+    name: flux-system
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  path: ./clusters/stand
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system

clusters/stand/flux-system/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- gotk-components.yaml
+- gotk-sync.yaml

clusters/stand/helm-repositories.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: yc-oci-charts
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 10m0s
+  url: oci://cr.yandex/crp3ccidau046kdj8g9q/charts
+  secretRef:
+    name: yc-cr-auth

clusters/stand/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./flux-system
+  - ./helm-repositories.yaml