.gitlab-ci.yml

@@ -4,7 +4,6 @@ include:
 
 stages:
   - build
-  - slices
 
 variables:
   RELEASE_NAME: iac-offline
@@ -34,18 +33,3 @@ build_offline_mirror:
     - docker push ${IMAGE_NAME}
     - docker push ${CR}/${RELEASE_NAME}:latest
   timeout: 30m
-
-publish_contour_slices:
-  stage: slices
-  image: ${CR}/infra/slice-builder:1.0.0
-  tags:
-    - build
-  rules:
-    - if: $CI_COMMIT_TAG =~ /^v?(?:\d+\.){2}\d+$/
-  variables:
-    GIT_DEPTH: "1"
-  script:
-    - printf '%s' "$SLICES_PUSHER_KEY" | base64 -d > /tmp/slices-key.json
-    - python3 ci/publish-slices.py --repo . --release "$CI_COMMIT_TAG" --commit "$CI_COMMIT_SHA" --key-file /tmp/slices-key.json
-  after_script:
-    - rm -f /tmp/slices-key.json

ci/build-slice.py

@@ -1,163 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import io
-import subprocess
-import sys
-import tarfile
-from pathlib import Path, PurePosixPath
-
-import yaml
-
-FLUX_KUSTOMIZATION = "kustomize.toolkit.fluxcd.io"
-KUSTOMIZATION_FILES = ("kustomization.yaml", "kustomization.yml", "Kustomization")
-
-
-class SliceError(Exception):
-    pass
-
-
-def is_remote(ref: str) -> bool:
-    return "://" in ref or ref.startswith(("git@", "github.com/", "gitlab.com/"))
-
-
-def kustomization_file(directory: Path) -> Path | None:
-    for name in KUSTOMIZATION_FILES:
-        candidate = directory / name
-        if candidate.is_file():
-            return candidate
-    return None
-
-
-def local_refs(doc: dict) -> list[str]:
-    refs = []
-    refs += doc.get("resources") or []
-    refs += doc.get("components") or []
-    refs += doc.get("patchesStrategicMerge") or []
-    refs += [p["path"] for p in doc.get("patches") or [] if isinstance(p, dict) and "path" in p]
-    for key in ("configMapGenerator", "secretGenerator"):
-        for gen in doc.get(key) or []:
-            for entry in (gen.get("files") or []) + (gen.get("envs") or []):
-                refs.append(entry.split("=", 1)[-1])
-            if gen.get("env"):
-                refs.append(gen["env"])
-    return [r for r in refs if isinstance(r, str)]
-
-
-def flux_paths(path: Path) -> list[str]:
-    try:
-        docs = list(yaml.safe_load_all(path.read_text()))
-    except yaml.YAMLError:
-        return []
-    return [
-        d["spec"]["path"]
-        for d in docs
-        if isinstance(d, dict)
-        and str(d.get("apiVersion", "")).startswith(FLUX_KUSTOMIZATION)
-        and d.get("kind") == "Kustomization"
-        and d.get("spec", {}).get("path")
-    ]
-
-
-def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]:
-    repo = repo.resolve()
-    files: set[PurePosixPath] = set()
-    entries: list[str] = []
-    seen_dirs: set[Path] = set()
-
-    def rel(p: Path) -> PurePosixPath:
-        try:
-            return PurePosixPath(p.resolve().relative_to(repo).as_posix())
-        except ValueError:
-            raise SliceError(f"ссылка за пределы репозитория: {p}")
-
-    def add_file(p: Path):
-        r = rel(p)
-        if r in files:
-            return
-        files.add(r)
-        if p.suffix in (".yaml", ".yml"):
-            for fp in flux_paths(p):
-                visit_dir(repo / fp, entry=True)
-
-    def visit_dir(d: Path, entry: bool = False):
-        d = d.resolve()
-        if d in seen_dirs:
-            return
-        seen_dirs.add(d)
-        kfile = kustomization_file(d)
-        if kfile is None:
-            raise SliceError(f"нет kustomization.yaml в {rel(d)}")
-        if entry:
-            entries.append(str(rel(d)))
-        add_file(kfile)
-        doc = yaml.safe_load(kfile.read_text()) or {}
-        for ref in local_refs(doc):
-            if is_remote(ref):
-                raise SliceError(f"удалённая ссылка {ref} в {rel(kfile)}")
-            target = (d / ref)
-            if target.is_dir():
-                visit_dir(target)
-            elif target.is_file():
-                add_file(target)
-            else:
-                raise SliceError(f"ссылка {ref} в {rel(kfile)} никуда не ведёт")
-
-    visit_dir(repo / start, entry=True)
-    return files, sorted(set(entries))
-
-
-def write_tar(repo: Path, files: set[PurePosixPath], out: Path):
-    buf = io.BytesIO()
-    with tarfile.open(fileobj=buf, mode="w", format=tarfile.PAX_FORMAT) as tar:
-        for f in sorted(files):
-            data = (repo / f).read_bytes()
-            info = tarfile.TarInfo(str(f))
-            info.size = len(data)
-            info.mode = 0o644
-            info.mtime = 0
-            info.uid = info.gid = 0
-            info.uname = info.gname = ""
-            tar.addfile(info, io.BytesIO(data))
-    out.write_bytes(buf.getvalue())
-
-
-def kustomize_build(path: Path) -> bytes:
-    p = subprocess.run(["kubectl", "kustomize", str(path)], capture_output=True)
-    if p.returncode != 0:
-        raise SliceError(f"kustomize build {path}: {p.stderr.decode().strip()}")
-    return p.stdout
-
-
-def verify(repo: Path, out: Path, entries: list[str]):
-    import tempfile
-    with tempfile.TemporaryDirectory() as tmp:
-        with tarfile.open(out) as tar:
-            tar.extractall(tmp, filter="data")
-        for e in entries:
-            if kustomize_build(repo / e) != kustomize_build(Path(tmp) / e):
-                raise SliceError(f"build {e} в срезе отличается от полного репозитория")
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("contour")
-    ap.add_argument("--repo", default=".")
-    ap.add_argument("--out", required=True)
-    ap.add_argument("--no-verify", action="store_true")
-    args = ap.parse_args()
-    repo = Path(args.repo).resolve()
-    out = Path(args.out)
-    try:
-        files, entries = collect(repo, f"clusters/{args.contour}")
-        write_tar(repo, files, out)
-        if not args.no_verify:
-            verify(repo, out, entries)
-    except SliceError as e:
-        print(f"срез {args.contour}: {e}", file=sys.stderr)
-        return 1
-    print(f"срез {args.contour}: {len(files)} файлов, слоёв {len(entries)}: {', '.join(entries)}")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

ci/publish-slices.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import base64
-import hashlib
-import importlib.util
-import json
-import os
-import re
-import sys
-import tempfile
-import urllib.error
-import urllib.parse
-import urllib.request
-from datetime import datetime, timezone
-from pathlib import Path
-
-import yaml
-
-spec = importlib.util.spec_from_file_location("build_slice", Path(__file__).with_name("build-slice.py"))
-bs = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(bs)
-
-REGISTRY_HOST = "cr.yandex"
-SLICE_REGISTRY = "crpct2b2lsmao4nm0cjk"
-INSTALLER_GITEA_URL = "http://gitea.gitea.svc:3000/infra/iac.git"
-CONFIG_TYPE = "application/vnd.sarex.slice.config.v1+json"
-LAYER_TYPE = "application/vnd.oci.image.layer.v1.tar"
-MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json"
-
-
-class PublishError(Exception):
-    pass
-
-
-def digest(data: bytes) -> str:
-    return "sha256:" + hashlib.sha256(data).hexdigest()
-
-
-def connected_contours(repo: Path) -> list[str]:
-    found = []
-    for sync in sorted(repo.glob("clusters/*/flux-system/gotk-sync.yaml")):
-        for doc in yaml.safe_load_all(sync.read_text()):
-            if isinstance(doc, dict) and doc.get("kind") == "GitRepository" and doc.get("spec", {}).get("url") == INSTALLER_GITEA_URL:
-                found.append(sync.parents[1].name)
-    return found
-
-
-def build_manifest(layer: bytes, annotations: dict[str, str]) -> tuple[bytes, bytes]:
-    config = b"{}"
-    manifest = {
-        "schemaVersion": 2,
-        "mediaType": MANIFEST_TYPE,
-        "config": {"mediaType": CONFIG_TYPE, "digest": digest(config), "size": len(config)},
-        "layers": [{
-            "mediaType": LAYER_TYPE,
-            "digest": digest(layer),
-            "size": len(layer),
-            "annotations": {"org.opencontainers.image.title": "slice.tar"},
-        }],
-        "annotations": dict(sorted(annotations.items())),
-    }
-    return config, json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode()
-
-
-class Registry:
-    def __init__(self, username: str, password: str):
-        self.base = f"https://{REGISTRY_HOST}"
-        self.basic = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
-        self.tokens: dict[str, str] = {}
-
-    def _send(self, method, url, data=None, headers=None):
-        req = urllib.request.Request(url, data=data, method=method, headers=headers or {})
-        try:
-            with urllib.request.urlopen(req, timeout=60) as r:
-                return r.status, dict(r.headers), r.read()
-        except urllib.error.HTTPError as e:
-            return e.code, dict(e.headers), e.read()
-
-    def _token(self, challenge: str, scope: str) -> str:
-        params = dict(re.findall(r'(\w+)="([^"]*)"', challenge))
-        realm = params.pop("realm")
-        params["scope"] = scope
-        status, _, body = self._send("GET", f"{realm}?{urllib.parse.urlencode(params)}", headers={"Authorization": self.basic})
-        if status != 200:
-            raise PublishError(f"авторизация в реестре: HTTP {status}")
-        payload = json.loads(body)
-        return payload.get("token") or payload.get("access_token")
-
-    def request(self, method, repo, path, data=None, headers=None):
-        scope = f"repository:{repo}:pull,push"
-        url = path if path.startswith("http") else f"{self.base}/v2/{repo}/{path}"
-        headers = dict(headers or {})
-        if scope in self.tokens:
-            headers["Authorization"] = f"Bearer {self.tokens[scope]}"
-        status, h, body = self._send(method, url, data, headers)
-        if status == 401:
-            challenge = {k.lower(): v for k, v in h.items()}.get("www-authenticate", "")
-            self.tokens[scope] = self._token(challenge, scope)
-            headers["Authorization"] = f"Bearer {self.tokens[scope]}"
-            status, h, body = self._send(method, url, data, headers)
-        return status, {k.lower(): v for k, v in h.items()}, body
-
-    def current_layer(self, repo: str) -> str | None:
-        status, _, body = self.request("GET", repo, "manifests/current", headers={"Accept": MANIFEST_TYPE})
-        if status == 404:
-            return None
-        if status != 200:
-            raise PublishError(f"{repo}:current: HTTP {status}")
-        return json.loads(body)["layers"][0]["digest"]
-
-    def put_blob(self, repo: str, data: bytes):
-        d = digest(data)
-        if self.request("HEAD", repo, f"blobs/{d}")[0] == 200:
-            return
-        status, h, _ = self.request("POST", repo, "blobs/uploads/")
-        if status != 202:
-            raise PublishError(f"загрузка блоба в {repo}: HTTP {status}")
-        location = h["location"]
-        if location.startswith("/"):
-            location = self.base + location
-        sep = "&" if "?" in location else "?"
-        status, _, _ = self.request("PUT", repo, f"{location}{sep}digest={d}", data, {"Content-Type": "application/octet-stream"})
-        if status != 201:
-            raise PublishError(f"загрузка блоба в {repo}: HTTP {status}")
-
-    def push(self, repo: str, tags: list[str], layer: bytes, annotations: dict[str, str]):
-        config, manifest = build_manifest(layer, annotations)
-        self.put_blob(repo, config)
-        self.put_blob(repo, layer)
-        for tag in tags:
-            status, _, body = self.request("PUT", repo, f"manifests/{tag}", manifest, {"Content-Type": MANIFEST_TYPE})
-            if status != 201:
-                raise PublishError(f"{repo}:{tag}: HTTP {status} {body[:200]!r}")
-
-
-def main() -> int:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--repo", default=".")
-    ap.add_argument("--release", required=True)
-    ap.add_argument("--commit", required=True)
-    ap.add_argument("--branch", default="")
-    ap.add_argument("--key-file", required=True)
-    args = ap.parse_args()
-    repo = Path(args.repo).resolve()
-    registry = Registry("json_key", Path(args.key_file).read_text())
-    contours = connected_contours(repo)
-    print(f"подключённые контуры: {', '.join(contours) or 'нет'}")
-    failed = []
-    for contour in contours:
-        target = f"{SLICE_REGISTRY}/{contour}-iac"
-        try:
-            with tempfile.TemporaryDirectory() as tmp:
-                out = Path(tmp) / "slice.tar"
-                files, entries = bs.collect(repo, f"clusters/{contour}")
-                bs.write_tar(repo, files, out)
-                bs.verify(repo, out, entries)
-                layer = out.read_bytes()
-            changed = registry.current_layer(target) != digest(layer)
-            tags = [args.commit[:8], args.release] + (["current"] if changed else [])
-            registry.push(target, tags, layer, {
-                "org.opencontainers.image.source": "iac",
-                "org.opencontainers.image.revision": args.commit,
-                "org.opencontainers.image.version": args.release,
-                "org.opencontainers.image.created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
-                "io.sarex.slice.branch": args.branch,
-                "io.sarex.slice.contour": contour,
-                "io.sarex.slice.kind": "iac",
-            })
-            state = "current переставлен" if changed else "содержимое как в current"
-            print(f"{target}: {len(files)} файлов, {digest(layer)[:19]}, теги {', '.join(tags)}, {state}")
-        except (bs.SliceError, PublishError) as e:
-            print(f"{target}: ОШИБКА {e}", file=sys.stderr)
-            failed.append(contour)
-    return 1 if failed else 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

ci/tests/test_build_slice.py

@@ -1,85 +0,0 @@
-import importlib.util
-import tarfile
-from pathlib import Path
-
-import pytest
-
-spec = importlib.util.spec_from_file_location("build_slice", Path(__file__).parents[1] / "build-slice.py")
-bs = importlib.util.module_from_spec(spec)
-spec.loader.exec_module(bs)
-
-
-def put(root: Path, rel: str, text: str):
-    p = root / rel
-    p.parent.mkdir(parents=True, exist_ok=True)
-    p.write_text(text)
-
-
-@pytest.fixture
-def repo(tmp_path):
-    put(tmp_path, "clusters/a/kustomization.yaml", "resources:\n  - ./flux-system\n  - ./layers.yaml\n")
-    put(tmp_path, "clusters/a/flux-system/kustomization.yaml", "resources:\n  - gotk-sync.yaml\n")
-    put(tmp_path, "clusters/a/flux-system/gotk-sync.yaml",
-        "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nspec:\n  path: ./clusters/a\n")
-    put(tmp_path, "clusters/a/layers.yaml",
-        "apiVersion: kustomize.toolkit.fluxcd.io/v1\nkind: Kustomization\nspec:\n  path: ./layers/a/00-vault\n")
-    put(tmp_path, "layers/a/00-vault/kustomization.yaml", "resources:\n  - ../../../infrastructure/vault/contour-template\n")
-    put(tmp_path, "infrastructure/vault/contour-template/kustomization.yaml",
-        "resources:\n  - ../base\npatches:\n  - path: vault.yaml\n")
-    put(tmp_path, "infrastructure/vault/contour-template/vault.yaml", "kind: HelmRelease\n")
-    put(tmp_path, "infrastructure/vault/base/kustomization.yaml", "resources:\n  - helmrelease.yaml\n")
-    put(tmp_path, "infrastructure/vault/base/helmrelease.yaml", "kind: HelmRelease\n")
-    put(tmp_path, "infrastructure/vault/base/unused.yaml", "kind: ConfigMap\n")
-    put(tmp_path, "infrastructure/vault/uralkal/kustomization.yaml", "resources:\n  - ../base\n")
-    put(tmp_path, "clusters/b/kustomization.yaml", "resources:\n  - ../../infrastructure/vault/uralkal\n")
-    return tmp_path
-
-
-def test_slice_follows_flux_paths_and_skips_foreign(repo):
-    files, entries = bs.collect(repo, "clusters/a")
-    names = {str(f) for f in files}
-    assert "infrastructure/vault/base/helmrelease.yaml" in names
-    assert "infrastructure/vault/contour-template/vault.yaml" in names
-    assert "layers/a/00-vault/kustomization.yaml" in names
-    assert not any("uralkal" in n or n.startswith("clusters/b") for n in names)
-    assert "infrastructure/vault/base/unused.yaml" not in names
-    assert entries == ["clusters/a", "layers/a/00-vault"]
-
-
-def test_remote_reference_stops_build(repo):
-    put(repo, "clusters/a/kustomization.yaml", "resources:\n  - https://github.com/x/y//z\n")
-    with pytest.raises(bs.SliceError, match="удалённая ссылка"):
-        bs.collect(repo, "clusters/a")
-
-
-def test_dangling_reference_stops_build(repo):
-    put(repo, "clusters/a/kustomization.yaml", "resources:\n  - ./missing.yaml\n")
-    with pytest.raises(bs.SliceError, match="никуда не ведёт"):
-        bs.collect(repo, "clusters/a")
-
-
-def test_tar_is_byte_identical_across_runs(repo, tmp_path_factory):
-    files, _ = bs.collect(repo, "clusters/a")
-    out1 = tmp_path_factory.mktemp("o") / "s.tar"
-    out2 = tmp_path_factory.mktemp("o") / "s.tar"
-    bs.write_tar(repo, files, out1)
-    (repo / "clusters/a/layers.yaml").touch()
-    bs.write_tar(repo, files, out2)
-    assert out1.read_bytes() == out2.read_bytes()
-    with tarfile.open(out1) as t:
-        assert all(m.mtime == 0 for m in t.getmembers())
-
-
-pspec = importlib.util.spec_from_file_location("publish_slices", Path(__file__).parents[1] / "publish-slices.py")
-ps = importlib.util.module_from_spec(pspec)
-pspec.loader.exec_module(ps)
-
-
-def test_only_installer_contours_are_connected(tmp_path):
-    put(tmp_path, "clusters/stand/flux-system/gotk-sync.yaml",
-        f"apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: {ps.INSTALLER_GITEA_URL}\n")
-    put(tmp_path, "clusters/vad/flux-system/gotk-sync.yaml",
-        "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: http://192.168.8.141:31361/infra/iac.git\n")
-    put(tmp_path, "clusters/old/flux-system/gotk-sync.yaml",
-        "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n  url: https://gitlab.sarex.io/infra/iac.git\n")
-    assert ps.connected_contours(tmp_path) == ["stand"]

clusters/stand/flux-system/gotk-sync.yaml

@@ -1,25 +0,0 @@
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: GitRepository
-metadata:
-  name: flux-system
-  namespace: flux-system
-spec:
-  interval: 1m0s
-  ref:
-    branch: main
-  url: http://gitea.gitea.svc:3000/infra/iac.git
-  secretRef:
-    name: flux-system
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: flux-system
-  namespace: flux-system
-spec:
-  interval: 10m0s
-  path: ./clusters/stand
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: flux-system

clusters/stand/flux-system/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-- gotk-components.yaml
-- gotk-sync.yaml

clusters/stand/helm-repositories.yaml

@@ -1,12 +0,0 @@
----
-apiVersion: source.toolkit.fluxcd.io/v1
-kind: HelmRepository
-metadata:
-  name: yc-oci-charts
-  namespace: flux-system
-spec:
-  type: oci
-  interval: 10m0s
-  url: oci://cr.yandex/crp3ccidau046kdj8g9q/charts
-  secretRef:
-    name: yc-cr-auth

clusters/stand/kustomization.yaml

@@ -1,5 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ./flux-system
-  - ./helm-repositories.yaml