++ slice jobs allow failure

.gitlab-ci.yml

@@ -5,6 +5,7 @@ include:
 stages:
   - build
   - slices
+  - slices-publish
 
 variables:
   RELEASE_NAME: iac-offline
@@ -35,7 +36,7 @@ build_offline_mirror:
     - docker push ${CR}/${RELEASE_NAME}:latest
   timeout: 30m
 
-publish_contour_slices:
+generate_slice_jobs:
   stage: slices
   image: ${CR}/infra/slice-builder:1.0.0
   tags:
@@ -45,7 +46,20 @@ publish_contour_slices:
   variables:
     GIT_DEPTH: "1"
   script:
-    - printf '%s' "$SLICES_PUSHER_KEY" | base64 -d > /tmp/slices-key.json
-    - python3 ci/publish-slices.py --repo . --release "$CI_COMMIT_TAG" --commit "$CI_COMMIT_SHA" --key-file /tmp/slices-key.json
-  after_script:
-    - rm -f /tmp/slices-key.json
+    - python3 ci/publish-slice.py generate --release "$CI_COMMIT_TAG" --commit "$CI_COMMIT_SHA" --out slices-pipeline.yml
+    - cat slices-pipeline.yml
+  artifacts:
+    paths:
+      - slices-pipeline.yml
+
+publish_contour_slices:
+  stage: slices-publish
+  rules:
+    - if: $CI_COMMIT_TAG =~ /^v?(?:\d+\.){2}\d+$/
+  needs:
+    - generate_slice_jobs
+  trigger:
+    include:
+      - artifact: slices-pipeline.yml
+        job: generate_slice_jobs
+    strategy: depend

ci/build-slice.py

@@ -68,7 +68,7 @@ def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]:
         try:
             return PurePosixPath(p.resolve().relative_to(repo).as_posix())
         except ValueError:
-            raise SliceError(f"ссылка за пределы репозитория: {p}")
+            raise SliceError(f"reference outside the repository: {p}")
 
     def add_file(p: Path):
         r = rel(p)
@@ -86,21 +86,21 @@ def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]:
         seen_dirs.add(d)
         kfile = kustomization_file(d)
         if kfile is None:
-            raise SliceError(f"нет kustomization.yaml в {rel(d)}")
+            raise SliceError(f"no kustomization.yaml in {rel(d)}")
         if entry:
             entries.append(str(rel(d)))
         add_file(kfile)
         doc = yaml.safe_load(kfile.read_text()) or {}
         for ref in local_refs(doc):
             if is_remote(ref):
-                raise SliceError(f"удалённая ссылка {ref} в {rel(kfile)}")
+                raise SliceError(f"remote reference {ref} in {rel(kfile)}")
             target = (d / ref)
             if target.is_dir():
                 visit_dir(target)
             elif target.is_file():
                 add_file(target)
             else:
-                raise SliceError(f"ссылка {ref} в {rel(kfile)} никуда не ведёт")
+                raise SliceError(f"reference {ref} in {rel(kfile)} does not exist")
 
     visit_dir(repo / start, entry=True)
     return files, sorted(set(entries))
@@ -124,7 +124,7 @@ def write_tar(repo: Path, files: set[PurePosixPath], out: Path):
 def kustomize_build(path: Path) -> bytes:
     p = subprocess.run(["kubectl", "kustomize", str(path)], capture_output=True)
     if p.returncode != 0:
-        raise SliceError(f"kustomize build {path}: {p.stderr.decode().strip()}")
+        raise SliceError(f"kustomize build {path} failed: {p.stderr.decode().strip()}")
     return p.stdout
 
 
@@ -135,7 +135,7 @@ def verify(repo: Path, out: Path, entries: list[str]):
             tar.extractall(tmp, filter="data")
         for e in entries:
             if kustomize_build(repo / e) != kustomize_build(Path(tmp) / e):
-                raise SliceError(f"build {e} в срезе отличается от полного репозитория")
+                raise SliceError(f"kustomize build of {e} differs between slice and full repository")
 
 
 def main() -> int:
@@ -153,9 +153,9 @@ def main() -> int:
         if not args.no_verify:
             verify(repo, out, entries)
     except SliceError as e:
-        print(f"срез {args.contour}: {e}", file=sys.stderr)
+        print(f"slice {args.contour}: {e}", file=sys.stderr)
         return 1
-    print(f"срез {args.contour}: {len(files)} файлов, слоёв {len(entries)}: {', '.join(entries)}")
+    print(f"slice {args.contour}: {len(files)} files, entry points: {', '.join(entries)}")
     return 0
 
 

ci/publish-slice.py

@@ -4,10 +4,10 @@ import base64
 import hashlib
 import importlib.util
 import json
-import os
 import re
 import sys
 import tempfile
+import time
 import urllib.error
 import urllib.parse
 import urllib.request
@@ -22,10 +22,10 @@ spec.loader.exec_module(bs)
 
 REGISTRY_HOST = "cr.yandex"
 SLICE_REGISTRY = "crpct2b2lsmao4nm0cjk"
-INSTALLER_GITEA_URL = "http://gitea.gitea.svc:3000/infra/iac.git"
 CONFIG_TYPE = "application/vnd.sarex.slice.config.v1+json"
 LAYER_TYPE = "application/vnd.oci.image.layer.v1.tar"
 MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json"
+JOB_IMAGE = "${CR}/infra/slice-builder:1.0.0"
 
 
 class PublishError(Exception):
@@ -36,13 +36,28 @@ def digest(data: bytes) -> str:
     return "sha256:" + hashlib.sha256(data).hexdigest()
 
 
-def connected_contours(repo: Path) -> list[str]:
-    found = []
-    for sync in sorted(repo.glob("clusters/*/flux-system/gotk-sync.yaml")):
-        for doc in yaml.safe_load_all(sync.read_text()):
-            if isinstance(doc, dict) and doc.get("kind") == "GitRepository" and doc.get("spec", {}).get("url") == INSTALLER_GITEA_URL:
-                found.append(sync.parents[1].name)
-    return found
+def list_contours(repo: Path) -> list[str]:
+    return sorted(p.parent.name for p in repo.glob("clusters/*/kustomization.yaml"))
+
+
+def child_pipeline(contours: list[str], release: str, commit: str) -> str:
+    jobs: dict = {"stages": ["slices"]}
+    for c in contours:
+        jobs[f"slice {c}"] = {
+            "stage": "slices",
+            "image": JOB_IMAGE,
+            "tags": ["build"],
+            "allow_failure": True,
+            "variables": {"GIT_DEPTH": "1"},
+            "script": [
+                "printf '%s' \"$SLICES_PUSHER_KEY\" | base64 -d > /tmp/slices-key.json",
+                f"python3 ci/publish-slice.py publish --contour {c} --release {release} --commit {commit} --key-file /tmp/slices-key.json",
+            ],
+            "after_script": ["rm -f /tmp/slices-key.json"],
+        }
+    if not contours:
+        jobs["no contours"] = {"stage": "slices", "image": JOB_IMAGE, "tags": ["build"], "script": ["echo 'no clusters/*/kustomization.yaml found'"]}
+    return yaml.safe_dump(jobs, sort_keys=False, allow_unicode=False)
 
 
 def build_manifest(layer: bytes, annotations: dict[str, str]) -> tuple[bytes, bytes]:
@@ -82,7 +97,7 @@ class Registry:
         params["scope"] = scope
         status, _, body = self._send("GET", f"{realm}?{urllib.parse.urlencode(params)}", headers={"Authorization": self.basic})
         if status != 200:
-            raise PublishError(f"авторизация в реестре: HTTP {status}")
+            raise PublishError(f"registry auth failed: HTTP {status}")
         payload = json.loads(body)
         return payload.get("token") or payload.get("access_token")
 
@@ -108,70 +123,94 @@ class Registry:
             raise PublishError(f"{repo}:current: HTTP {status}")
         return json.loads(body)["layers"][0]["digest"]
 
-    def put_blob(self, repo: str, data: bytes):
+    def put_blob(self, repo: str, data: bytes) -> bool:
         d = digest(data)
         if self.request("HEAD", repo, f"blobs/{d}")[0] == 200:
-            return
+            return False
         status, h, _ = self.request("POST", repo, "blobs/uploads/")
         if status != 202:
-            raise PublishError(f"загрузка блоба в {repo}: HTTP {status}")
+            raise PublishError(f"blob upload to {repo}: HTTP {status}")
         location = h["location"]
         if location.startswith("/"):
             location = self.base + location
         sep = "&" if "?" in location else "?"
         status, _, _ = self.request("PUT", repo, f"{location}{sep}digest={d}", data, {"Content-Type": "application/octet-stream"})
         if status != 201:
-            raise PublishError(f"загрузка блоба в {repo}: HTTP {status}")
+            raise PublishError(f"blob upload to {repo}: HTTP {status}")
+        return True
 
-    def push(self, repo: str, tags: list[str], layer: bytes, annotations: dict[str, str]):
+    def push(self, repo: str, tags: list[str], layer: bytes, annotations: dict[str, str]) -> bool:
         config, manifest = build_manifest(layer, annotations)
         self.put_blob(repo, config)
-        self.put_blob(repo, layer)
+        uploaded = self.put_blob(repo, layer)
         for tag in tags:
             status, _, body = self.request("PUT", repo, f"manifests/{tag}", manifest, {"Content-Type": MANIFEST_TYPE})
             if status != 201:
                 raise PublishError(f"{repo}:{tag}: HTTP {status} {body[:200]!r}")
+        return uploaded
+
+
+def log(msg: str):
+    print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
+
+
+def publish(repo: Path, contour: str, release: str, commit: str, key_file: Path) -> int:
+    target = f"{SLICE_REGISTRY}/{contour}-iac"
+    log(f"contour {contour}, source iac@{commit[:8]}, release {release}")
+    try:
+        with tempfile.TemporaryDirectory() as tmp:
+            out = Path(tmp) / "slice.tar"
+            files, entries = bs.collect(repo, f"clusters/{contour}")
+            log(f"collected {len(files)} files, entry points: {', '.join(entries)}")
+            bs.write_tar(repo, files, out)
+            bs.verify(repo, out, entries)
+            log(f"verified: kustomize build of {len(entries)} entry point(s) identical to full repository")
+            layer = out.read_bytes()
+        new = digest(layer)
+        log(f"slice size {len(layer)} bytes, digest {new}")
+        registry = Registry("json_key", key_file.read_text())
+        current = registry.current_layer(f"{target}")
+        changed = current != new
+        log(f"registry {REGISTRY_HOST}/{target}, current " + (current or "absent") + (", content changed" if changed else ", content unchanged"))
+        tags = [commit[:8], release] + (["current"] if changed else [])
+        uploaded = registry.push(target, tags, layer, {
+            "org.opencontainers.image.source": "iac",
+            "org.opencontainers.image.revision": commit,
+            "org.opencontainers.image.version": release,
+            "org.opencontainers.image.created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+            "io.sarex.slice.contour": contour,
+            "io.sarex.slice.kind": "iac",
+        })
+        log(f"layer {'uploaded' if uploaded else 'already in registry'}, tags pushed: {', '.join(tags)}")
+    except (bs.SliceError, PublishError) as e:
+        log(f"FAILED: {e}")
+        return 1
+    log("done")
+    return 0
 
 
 def main() -> int:
     ap = argparse.ArgumentParser()
-    ap.add_argument("--repo", default=".")
-    ap.add_argument("--release", required=True)
-    ap.add_argument("--commit", required=True)
-    ap.add_argument("--branch", default="")
-    ap.add_argument("--key-file", required=True)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    gen = sub.add_parser("generate")
+    gen.add_argument("--repo", default=".")
+    gen.add_argument("--release", required=True)
+    gen.add_argument("--commit", required=True)
+    gen.add_argument("--out", required=True)
+    pub = sub.add_parser("publish")
+    pub.add_argument("--repo", default=".")
+    pub.add_argument("--contour", required=True)
+    pub.add_argument("--release", required=True)
+    pub.add_argument("--commit", required=True)
+    pub.add_argument("--key-file", required=True)
     args = ap.parse_args()
     repo = Path(args.repo).resolve()
-    registry = Registry("json_key", Path(args.key_file).read_text())
-    contours = connected_contours(repo)
-    print(f"подключённые контуры: {', '.join(contours) or 'нет'}")
-    failed = []
-    for contour in contours:
-        target = f"{SLICE_REGISTRY}/{contour}-iac"
-        try:
-            with tempfile.TemporaryDirectory() as tmp:
-                out = Path(tmp) / "slice.tar"
-                files, entries = bs.collect(repo, f"clusters/{contour}")
-                bs.write_tar(repo, files, out)
-                bs.verify(repo, out, entries)
-                layer = out.read_bytes()
-            changed = registry.current_layer(target) != digest(layer)
-            tags = [args.commit[:8], args.release] + (["current"] if changed else [])
-            registry.push(target, tags, layer, {
-                "org.opencontainers.image.source": "iac",
-                "org.opencontainers.image.revision": args.commit,
-                "org.opencontainers.image.version": args.release,
-                "org.opencontainers.image.created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                "io.sarex.slice.branch": args.branch,
-                "io.sarex.slice.contour": contour,
-                "io.sarex.slice.kind": "iac",
-            })
-            state = "current переставлен" if changed else "содержимое как в current"
-            print(f"{target}: {len(files)} файлов, {digest(layer)[:19]}, теги {', '.join(tags)}, {state}")
-        except (bs.SliceError, PublishError) as e:
-            print(f"{target}: ОШИБКА {e}", file=sys.stderr)
-            failed.append(contour)
-    return 1 if failed else 0
+    if args.cmd == "generate":
+        contours = list_contours(repo)
+        Path(args.out).write_text(child_pipeline(contours, args.release, args.commit))
+        log(f"{len(contours)} contours: {', '.join(contours)}")
+        return 0
+    return publish(repo, args.contour, args.release, args.commit, Path(args.key_file))
 
 
 if __name__ == "__main__":

ci/tests/test_build_slice.py

@@ -3,6 +3,7 @@ import tarfile
 from pathlib import Path
 
 import pytest
+import yaml
 
 spec = importlib.util.spec_from_file_location("build_slice", Path(__file__).parents[1] / "build-slice.py")
 bs = importlib.util.module_from_spec(spec)
@@ -48,13 +49,13 @@ def test_slice_follows_flux_paths_and_skips_foreign(repo):
 
 def test_remote_reference_stops_build(repo):
     put(repo, "clusters/a/kustomization.yaml", "resources:\n  - https://github.com/x/y//z\n")
-    with pytest.raises(bs.SliceError, match="удалённая ссылка"):
+    with pytest.raises(bs.SliceError, match="remote reference"):
         bs.collect(repo, "clusters/a")
 
 
 def test_dangling_reference_stops_build(repo):
     put(repo, "clusters/a/kustomization.yaml", "resources:\n  - ./missing.yaml\n")
-    with pytest.raises(bs.SliceError, match="никуда не ведёт"):
+    with pytest.raises(bs.SliceError, match="does not exist"):
         bs.collect(repo, "clusters/a")
 
 
@@ -70,16 +71,23 @@ def test_tar_is_byte_identical_across_runs(repo, tmp_path_factory):
         assert all(m.mtime == 0 for m in t.getmembers())
 
 
-pspec = importlib.util.spec_from_file_location("publish_slices", Path(__file__).parents[1] / "publish-slices.py")
+
+pspec = importlib.util.spec_from_file_location("publish_slice", Path(__file__).parents[1] / "publish-slice.py")
 ps = importlib.util.module_from_spec(pspec)
 pspec.loader.exec_module(ps)
 
 
-def test_only_installer_contours_are_connected(tmp_path):
-    put(tmp_path, "clusters/stand/flux-system/gotk-sync.yaml",
-        f"apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: {ps.INSTALLER_GITEA_URL}\n")
-    put(tmp_path, "clusters/vad/flux-system/gotk-sync.yaml",
-        "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: http://192.168.8.141:31361/infra/iac.git\n")
-    put(tmp_path, "clusters/old/flux-system/gotk-sync.yaml",
-        "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: https://gitlab.sarex.io/infra/iac.git\n")
-    assert ps.connected_contours(tmp_path) == ["stand"]
+def test_every_cluster_with_kustomization_is_a_contour(tmp_path):
+    put(tmp_path, "clusters/stand/kustomization.yaml", "resources: []\n")
+    put(tmp_path, "clusters/vad/kustomization.yaml", "resources: []\n")
+    put(tmp_path, "clusters/contour/apps.yaml", "kind: Kustomization\n")
+    assert ps.list_contours(tmp_path) == ["stand", "vad"]
+
+
+def test_child_pipeline_has_one_job_per_contour():
+    doc = yaml.safe_load(ps.child_pipeline(["stand", "vad"], "3.9.68", "abcdef1234"))
+    jobs = {k: v for k, v in doc.items() if k != "stages"}
+    assert sorted(jobs) == ["slice stand", "slice vad"]
+    assert all(j["allow_failure"] for j in jobs.values())
+    assert "--contour vad --release 3.9.68 --commit abcdef1234" in jobs["slice vad"]["script"][1]
+    assert all(ord(ch) < 128 for ch in ps.child_pipeline(["stand"], "1.0.0", "a" * 40))