diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52df9f6..97ed635 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ include: stages: - build - slices + - slices-publish variables: RELEASE_NAME: iac-offline @@ -35,7 +36,7 @@ build_offline_mirror: - docker push ${CR}/${RELEASE_NAME}:latest timeout: 30m -publish_contour_slices: +generate_slice_jobs: stage: slices image: ${CR}/infra/slice-builder:1.0.0 tags: @@ -45,7 +46,20 @@ publish_contour_slices: variables: GIT_DEPTH: "1" script: - - printf '%s' "$SLICES_PUSHER_KEY" | base64 -d > /tmp/slices-key.json - - python3 ci/publish-slices.py --repo . --release "$CI_COMMIT_TAG" --commit "$CI_COMMIT_SHA" --key-file /tmp/slices-key.json - after_script: - - rm -f /tmp/slices-key.json + - python3 ci/publish-slice.py generate --release "$CI_COMMIT_TAG" --commit "$CI_COMMIT_SHA" --out slices-pipeline.yml + - cat slices-pipeline.yml + artifacts: + paths: + - slices-pipeline.yml + +publish_contour_slices: + stage: slices-publish + rules: + - if: $CI_COMMIT_TAG =~ /^v?(?:\d+\.){2}\d+$/ + needs: + - generate_slice_jobs + trigger: + include: + - artifact: slices-pipeline.yml + job: generate_slice_jobs + strategy: depend diff --git a/ci/build-slice.py b/ci/build-slice.py index 277f0be..aa7bbd3 100644 --- a/ci/build-slice.py +++ b/ci/build-slice.py @@ -68,7 +68,7 @@ def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]: try: return PurePosixPath(p.resolve().relative_to(repo).as_posix()) except ValueError: - raise SliceError(f"ссылка за пределы репозитория: {p}") + raise SliceError(f"reference outside the repository: {p}") def add_file(p: Path): r = rel(p) @@ -86,21 +86,21 @@ def collect(repo: Path, start: str) -> tuple[set[PurePosixPath], list[str]]: seen_dirs.add(d) kfile = kustomization_file(d) if kfile is None: - raise SliceError(f"нет kustomization.yaml в {rel(d)}") + raise SliceError(f"no kustomization.yaml in {rel(d)}") if entry: entries.append(str(rel(d))) add_file(kfile) doc = yaml.safe_load(kfile.read_text()) or {} for ref in local_refs(doc): if is_remote(ref): - raise SliceError(f"удалённая ссылка {ref} в {rel(kfile)}") + raise SliceError(f"remote reference {ref} in {rel(kfile)}") target = (d / ref) if target.is_dir(): visit_dir(target) elif target.is_file(): add_file(target) else: - raise SliceError(f"ссылка {ref} в {rel(kfile)} никуда не ведёт") + raise SliceError(f"reference {ref} in {rel(kfile)} does not exist") visit_dir(repo / start, entry=True) return files, sorted(set(entries)) @@ -124,7 +124,7 @@ def write_tar(repo: Path, files: set[PurePosixPath], out: Path): def kustomize_build(path: Path) -> bytes: p = subprocess.run(["kubectl", "kustomize", str(path)], capture_output=True) if p.returncode != 0: - raise SliceError(f"kustomize build {path}: {p.stderr.decode().strip()}") + raise SliceError(f"kustomize build {path} failed: {p.stderr.decode().strip()}") return p.stdout @@ -135,7 +135,7 @@ def verify(repo: Path, out: Path, entries: list[str]): tar.extractall(tmp, filter="data") for e in entries: if kustomize_build(repo / e) != kustomize_build(Path(tmp) / e): - raise SliceError(f"build {e} в срезе отличается от полного репозитория") + raise SliceError(f"kustomize build of {e} differs between slice and full repository") def main() -> int: @@ -153,9 +153,9 @@ def main() -> int: if not args.no_verify: verify(repo, out, entries) except SliceError as e: - print(f"срез {args.contour}: {e}", file=sys.stderr) + print(f"slice {args.contour}: {e}", file=sys.stderr) return 1 - print(f"срез {args.contour}: {len(files)} файлов, слоёв {len(entries)}: {', '.join(entries)}") + print(f"slice {args.contour}: {len(files)} files, entry points: {', '.join(entries)}") return 0 diff --git a/ci/publish-slices.py b/ci/publish-slice.py similarity index 52% rename from ci/publish-slices.py rename to ci/publish-slice.py index c4cf212..3210277 100644 --- a/ci/publish-slices.py +++ b/ci/publish-slice.py @@ -4,10 +4,10 @@ import base64 import hashlib import importlib.util import json -import os import re import sys import tempfile +import time import urllib.error import urllib.parse import urllib.request @@ -22,10 +22,10 @@ spec.loader.exec_module(bs) REGISTRY_HOST = "cr.yandex" SLICE_REGISTRY = "crpct2b2lsmao4nm0cjk" -INSTALLER_GITEA_URL = "http://gitea.gitea.svc:3000/infra/iac.git" CONFIG_TYPE = "application/vnd.sarex.slice.config.v1+json" LAYER_TYPE = "application/vnd.oci.image.layer.v1.tar" MANIFEST_TYPE = "application/vnd.oci.image.manifest.v1+json" +JOB_IMAGE = "${CR}/infra/slice-builder:1.0.0" class PublishError(Exception): @@ -36,13 +36,27 @@ def digest(data: bytes) -> str: return "sha256:" + hashlib.sha256(data).hexdigest() -def connected_contours(repo: Path) -> list[str]: - found = [] - for sync in sorted(repo.glob("clusters/*/flux-system/gotk-sync.yaml")): - for doc in yaml.safe_load_all(sync.read_text()): - if isinstance(doc, dict) and doc.get("kind") == "GitRepository" and doc.get("spec", {}).get("url") == INSTALLER_GITEA_URL: - found.append(sync.parents[1].name) - return found +def list_contours(repo: Path) -> list[str]: + return sorted(p.parent.name for p in repo.glob("clusters/*/kustomization.yaml")) + + +def child_pipeline(contours: list[str], release: str, commit: str) -> str: + jobs: dict = {"stages": ["slices"]} + for c in contours: + jobs[f"slice {c}"] = { + "stage": "slices", + "image": JOB_IMAGE, + "tags": ["build"], + "variables": {"GIT_DEPTH": "1"}, + "script": [ + "printf '%s' \"$SLICES_PUSHER_KEY\" | base64 -d > /tmp/slices-key.json", + f"python3 ci/publish-slice.py publish --contour {c} --release {release} --commit {commit} --key-file /tmp/slices-key.json", + ], + "after_script": ["rm -f /tmp/slices-key.json"], + } + if not contours: + jobs["no contours"] = {"stage": "slices", "image": JOB_IMAGE, "tags": ["build"], "script": ["echo 'no clusters/*/kustomization.yaml found'"]} + return yaml.safe_dump(jobs, sort_keys=False, allow_unicode=False) def build_manifest(layer: bytes, annotations: dict[str, str]) -> tuple[bytes, bytes]: @@ -82,7 +96,7 @@ class Registry: params["scope"] = scope status, _, body = self._send("GET", f"{realm}?{urllib.parse.urlencode(params)}", headers={"Authorization": self.basic}) if status != 200: - raise PublishError(f"авторизация в реестре: HTTP {status}") + raise PublishError(f"registry auth failed: HTTP {status}") payload = json.loads(body) return payload.get("token") or payload.get("access_token") @@ -108,70 +122,94 @@ class Registry: raise PublishError(f"{repo}:current: HTTP {status}") return json.loads(body)["layers"][0]["digest"] - def put_blob(self, repo: str, data: bytes): + def put_blob(self, repo: str, data: bytes) -> bool: d = digest(data) if self.request("HEAD", repo, f"blobs/{d}")[0] == 200: - return + return False status, h, _ = self.request("POST", repo, "blobs/uploads/") if status != 202: - raise PublishError(f"загрузка блоба в {repo}: HTTP {status}") + raise PublishError(f"blob upload to {repo}: HTTP {status}") location = h["location"] if location.startswith("/"): location = self.base + location sep = "&" if "?" in location else "?" status, _, _ = self.request("PUT", repo, f"{location}{sep}digest={d}", data, {"Content-Type": "application/octet-stream"}) if status != 201: - raise PublishError(f"загрузка блоба в {repo}: HTTP {status}") + raise PublishError(f"blob upload to {repo}: HTTP {status}") + return True - def push(self, repo: str, tags: list[str], layer: bytes, annotations: dict[str, str]): + def push(self, repo: str, tags: list[str], layer: bytes, annotations: dict[str, str]) -> bool: config, manifest = build_manifest(layer, annotations) self.put_blob(repo, config) - self.put_blob(repo, layer) + uploaded = self.put_blob(repo, layer) for tag in tags: status, _, body = self.request("PUT", repo, f"manifests/{tag}", manifest, {"Content-Type": MANIFEST_TYPE}) if status != 201: raise PublishError(f"{repo}:{tag}: HTTP {status} {body[:200]!r}") + return uploaded + + +def log(msg: str): + print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True) + + +def publish(repo: Path, contour: str, release: str, commit: str, key_file: Path) -> int: + target = f"{SLICE_REGISTRY}/{contour}-iac" + log(f"contour {contour}, source iac@{commit[:8]}, release {release}") + try: + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "slice.tar" + files, entries = bs.collect(repo, f"clusters/{contour}") + log(f"collected {len(files)} files, entry points: {', '.join(entries)}") + bs.write_tar(repo, files, out) + bs.verify(repo, out, entries) + log(f"verified: kustomize build of {len(entries)} entry point(s) identical to full repository") + layer = out.read_bytes() + new = digest(layer) + log(f"slice size {len(layer)} bytes, digest {new}") + registry = Registry("json_key", key_file.read_text()) + current = registry.current_layer(f"{target}") + changed = current != new + log(f"registry {REGISTRY_HOST}/{target}, current " + (current or "absent") + (", content changed" if changed else ", content unchanged")) + tags = [commit[:8], release] + (["current"] if changed else []) + uploaded = registry.push(target, tags, layer, { + "org.opencontainers.image.source": "iac", + "org.opencontainers.image.revision": commit, + "org.opencontainers.image.version": release, + "org.opencontainers.image.created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "io.sarex.slice.contour": contour, + "io.sarex.slice.kind": "iac", + }) + log(f"layer {'uploaded' if uploaded else 'already in registry'}, tags pushed: {', '.join(tags)}") + except (bs.SliceError, PublishError) as e: + log(f"FAILED: {e}") + return 1 + log("done") + return 0 def main() -> int: ap = argparse.ArgumentParser() - ap.add_argument("--repo", default=".") - ap.add_argument("--release", required=True) - ap.add_argument("--commit", required=True) - ap.add_argument("--branch", default="") - ap.add_argument("--key-file", required=True) + sub = ap.add_subparsers(dest="cmd", required=True) + gen = sub.add_parser("generate") + gen.add_argument("--repo", default=".") + gen.add_argument("--release", required=True) + gen.add_argument("--commit", required=True) + gen.add_argument("--out", required=True) + pub = sub.add_parser("publish") + pub.add_argument("--repo", default=".") + pub.add_argument("--contour", required=True) + pub.add_argument("--release", required=True) + pub.add_argument("--commit", required=True) + pub.add_argument("--key-file", required=True) args = ap.parse_args() repo = Path(args.repo).resolve() - registry = Registry("json_key", Path(args.key_file).read_text()) - contours = connected_contours(repo) - print(f"подключённые контуры: {', '.join(contours) or 'нет'}") - failed = [] - for contour in contours: - target = f"{SLICE_REGISTRY}/{contour}-iac" - try: - with tempfile.TemporaryDirectory() as tmp: - out = Path(tmp) / "slice.tar" - files, entries = bs.collect(repo, f"clusters/{contour}") - bs.write_tar(repo, files, out) - bs.verify(repo, out, entries) - layer = out.read_bytes() - changed = registry.current_layer(target) != digest(layer) - tags = [args.commit[:8], args.release] + (["current"] if changed else []) - registry.push(target, tags, layer, { - "org.opencontainers.image.source": "iac", - "org.opencontainers.image.revision": args.commit, - "org.opencontainers.image.version": args.release, - "org.opencontainers.image.created": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "io.sarex.slice.branch": args.branch, - "io.sarex.slice.contour": contour, - "io.sarex.slice.kind": "iac", - }) - state = "current переставлен" if changed else "содержимое как в current" - print(f"{target}: {len(files)} файлов, {digest(layer)[:19]}, теги {', '.join(tags)}, {state}") - except (bs.SliceError, PublishError) as e: - print(f"{target}: ОШИБКА {e}", file=sys.stderr) - failed.append(contour) - return 1 if failed else 0 + if args.cmd == "generate": + contours = list_contours(repo) + Path(args.out).write_text(child_pipeline(contours, args.release, args.commit)) + log(f"{len(contours)} contours: {', '.join(contours)}") + return 0 + return publish(repo, args.contour, args.release, args.commit, Path(args.key_file)) if __name__ == "__main__": diff --git a/ci/tests/test_build_slice.py b/ci/tests/test_build_slice.py index aaafc44..2436bc9 100644 --- a/ci/tests/test_build_slice.py +++ b/ci/tests/test_build_slice.py @@ -3,6 +3,7 @@ import tarfile from pathlib import Path import pytest +import yaml spec = importlib.util.spec_from_file_location("build_slice", Path(__file__).parents[1] / "build-slice.py") bs = importlib.util.module_from_spec(spec) @@ -48,13 +49,13 @@ def test_slice_follows_flux_paths_and_skips_foreign(repo): def test_remote_reference_stops_build(repo): put(repo, "clusters/a/kustomization.yaml", "resources:\n - https://github.com/x/y//z\n") - with pytest.raises(bs.SliceError, match="удалённая ссылка"): + with pytest.raises(bs.SliceError, match="remote reference"): bs.collect(repo, "clusters/a") def test_dangling_reference_stops_build(repo): put(repo, "clusters/a/kustomization.yaml", "resources:\n - ./missing.yaml\n") - with pytest.raises(bs.SliceError, match="никуда не ведёт"): + with pytest.raises(bs.SliceError, match="does not exist"): bs.collect(repo, "clusters/a") @@ -70,16 +71,22 @@ def test_tar_is_byte_identical_across_runs(repo, tmp_path_factory): assert all(m.mtime == 0 for m in t.getmembers()) -pspec = importlib.util.spec_from_file_location("publish_slices", Path(__file__).parents[1] / "publish-slices.py") + +pspec = importlib.util.spec_from_file_location("publish_slice", Path(__file__).parents[1] / "publish-slice.py") ps = importlib.util.module_from_spec(pspec) pspec.loader.exec_module(ps) -def test_only_installer_contours_are_connected(tmp_path): - put(tmp_path, "clusters/stand/flux-system/gotk-sync.yaml", - f"apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n url: {ps.INSTALLER_GITEA_URL}\n") - put(tmp_path, "clusters/vad/flux-system/gotk-sync.yaml", - "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n url: http://192.168.8.141:31361/infra/iac.git\n") - put(tmp_path, "clusters/old/flux-system/gotk-sync.yaml", - "apiVersion: source.toolkit.fluxcd.io/v1\nkind: GitRepository\nspec:\n url: https://gitlab.sarex.io/infra/iac.git\n") - assert ps.connected_contours(tmp_path) == ["stand"] +def test_every_cluster_with_kustomization_is_a_contour(tmp_path): + put(tmp_path, "clusters/stand/kustomization.yaml", "resources: []\n") + put(tmp_path, "clusters/vad/kustomization.yaml", "resources: []\n") + put(tmp_path, "clusters/contour/apps.yaml", "kind: Kustomization\n") + assert ps.list_contours(tmp_path) == ["stand", "vad"] + + +def test_child_pipeline_has_one_job_per_contour(): + doc = yaml.safe_load(ps.child_pipeline(["stand", "vad"], "3.9.68", "abcdef1234")) + jobs = {k: v for k, v in doc.items() if k != "stages"} + assert sorted(jobs) == ["slice stand", "slice vad"] + assert "--contour vad --release 3.9.68 --commit abcdef1234" in jobs["slice vad"]["script"][1] + assert all(ord(ch) < 128 for ch in ps.child_pipeline(["stand"], "1.0.0", "a" * 40))