bastion agent 0.1.16
This commit is contained in:
Privatae
@@ -0,0 +1,116 @@
|
||||
# The checks of .gitea/workflows/ci.yml, for the copy of this repository on
|
||||
# GitHub. Gitea reads .gitea/workflows when it exists and then ignores this
|
||||
# directory, so each forge runs one of the two files, never both.
|
||||
#
|
||||
# The jobs, images and commands are the same as there. No action from the
|
||||
# marketplace is used: the commit is fetched with git from this repository's
|
||||
# public URL, so nothing outside this file and the pinned images runs. The job
|
||||
# token can only read.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
PY_FLOOR: python@sha256:1d52838af602b4b5a831beb13a0e4d073280665ea7be7f69ce2382f29c5a613f # 3.8-slim
|
||||
PY_CURRENT: python@sha256:7c61056e61ac89e852de05f3dc6fa51a6dd2181797bceed46aa725dd7cb2cd3b # 3.13-slim
|
||||
# The PyYAML release bastion_agent/build.py vendors; the manifest loader imports it.
|
||||
PYYAML: pyyaml==6.0.3
|
||||
# The linter: every name a module imports is used, and none is undefined.
|
||||
PYFLAKES: pyflakes==4.0.3
|
||||
# The pull request's own head, or the pushed commit.
|
||||
WANT: ${{ github.event.pull_request.head.sha || github.sha }}
|
||||
REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: fetch the commit
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init -q .
|
||||
git fetch -q --depth=1 "$REPO_URL" "$WANT"
|
||||
git checkout -q --detach FETCH_HEAD
|
||||
test "$(git rev-parse HEAD)" = "$WANT"
|
||||
- name: the 3.8 syntax floor
|
||||
run: |
|
||||
docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
|
||||
python -m unittest -v test_syntax_floor
|
||||
- name: pyflakes
|
||||
run: |
|
||||
docker run --rm -v "$PWD":/src:ro -w /src -e PYFLAKES "$PY_CURRENT" sh -euc '
|
||||
pip install -q --disable-pip-version-check --root-user-action=ignore "$PYFLAKES"
|
||||
python -m pyflakes bastion_agent pii scripts test_syntax_floor.py'
|
||||
- name: every test, on Python 3.8 and on a current Python
|
||||
# age is installed so the seal round trips run, and git so the release
|
||||
# verifier's end-to-end tests do. Every test runs here: a skipped one
|
||||
# fails the job (the only skips the suite has are for Windows, or for a
|
||||
# tool missing, and this job installs the tools).
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for img in "$PY_FLOOR" "$PY_CURRENT"; do
|
||||
docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
|
||||
python --version
|
||||
apt-get -qq update >/dev/null
|
||||
apt-get -qq install -y --no-install-recommends age git >/dev/null
|
||||
pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
|
||||
python -m unittest discover -v . > /tmp/tests.log 2>&1 || { cat /tmp/tests.log; exit 1; }
|
||||
grep -E "^(Ran|OK|FAILED)" /tmp/tests.log
|
||||
if grep -E "\.\.\. skipped|^(OK|FAILED) \(.*skipped=" /tmp/tests.log; then echo "a test was skipped; CI runs every test"; exit 1; fi'
|
||||
done
|
||||
|
||||
personal-data:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: fetch the commit
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init -q .
|
||||
git fetch -q --depth=1 "$REPO_URL" "$WANT"
|
||||
git checkout -q --detach FETCH_HEAD
|
||||
test "$(git rev-parse HEAD)" = "$WANT"
|
||||
- name: no person's data in the tree
|
||||
# Reads files, runs nothing from the tree. The term check needs the
|
||||
# maintainers' key and the entity-id check the reserved ids, which
|
||||
# this job never holds, so both are skipped and the run prints a NOTE
|
||||
# for each; the provenance, email and phone checks run.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker run --rm -v "$PWD":/src:ro -w /src "$PY_CURRENT" \
|
||||
python -I scripts/test_no_personal_data.py --root /src
|
||||
|
||||
reproducible:
|
||||
needs: test
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- name: fetch the commit
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git init -q .
|
||||
git fetch -q --depth=1 "$REPO_URL" "$WANT"
|
||||
git checkout -q --detach FETCH_HEAD
|
||||
test "$(git rev-parse HEAD)" = "$WANT"
|
||||
- name: two builds on each Python produce one sha256
|
||||
# Anyone must be able to rebuild a release and get the published bytes,
|
||||
# so the .pyz may not depend on the build or the Python that made it.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for img in "$PY_FLOOR" "$PY_CURRENT"; do
|
||||
docker run --rm -v "$PWD":/src:ro -w /src -e PYYAML "$img" sh -euc '
|
||||
pip install -q --disable-pip-version-check --root-user-action=ignore "$PYYAML"
|
||||
python -m bastion_agent.build --out /tmp/a | awk "{print \$NF}"
|
||||
python -m bastion_agent.build --out /tmp/b | awk "{print \$NF}"'
|
||||
done | tee digests
|
||||
test "$(wc -l < digests)" -eq 4
|
||||
test "$(sort -u digests | wc -l)" -eq 1
|
||||
echo "every build is $(head -1 digests)"
|
||||
@@ -0,0 +1,174 @@
|
||||
# Attach the release files to the GitHub release of a tag v<version>.
|
||||
#
|
||||
# GitHub only: Gitea reads .gitea/workflows when it exists and then ignores
|
||||
# this directory, so the forge's copy of this repository never runs this file.
|
||||
#
|
||||
# The files are not uploaded from anywhere else. This job rebuilds them from
|
||||
# the tag and attaches them only when they are the published release:
|
||||
#
|
||||
# 1. bastion_agent/verify_release.py fetches the live manifest
|
||||
# (privatae.ai/bastion-agent/version.json) and its signature, checks the
|
||||
# signature against the release keys the bastion pins, refuses unless the
|
||||
# manifest names this tag's version and the tag is annotated with the line
|
||||
# `built from <built_from>`, rebuilds the .pyz from the tag with that
|
||||
# tag's own build.py, and compares the sha256 with the manifest;
|
||||
# 2. only then are the rebuilt .pyz, its .sha256, the manifest (its exact
|
||||
# bytes) and the signature file attached to the release, which is created
|
||||
# when the tag has none, with that version's CHANGELOG section.
|
||||
#
|
||||
# The live manifest names only the newest release, so a release can be given
|
||||
# its files only while it is the published one. Older releases, whose manifest
|
||||
# is no longer served, cannot be filled in afterwards. A run that failed (the
|
||||
# tag pushed before the release went live, say) is run again from the Actions
|
||||
# tab with the tag as input.
|
||||
#
|
||||
# No action from the marketplace is used, and nothing is fetched with a
|
||||
# credential: the repository is read from its public URL. The job may write
|
||||
# to the repository's releases; the token is handed to the last step only.
|
||||
name: release
|
||||
|
||||
on:
|
||||
push:
|
||||
tags: ["v*.*.*"]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "the release tag, vX.Y.Z"
|
||||
required: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
REPO_URL: ${{ github.server_url }}/${{ github.repository }}.git
|
||||
TAG: ${{ github.event.inputs.tag || github.ref_name }}
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -euo pipefail {0}
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- name: the tag is a release tag
|
||||
run: |
|
||||
printf '%s\n' "$TAG" | grep -Exq 'v[0-9]+\.[0-9]+\.[0-9]+' \
|
||||
|| { echo "not a release tag: $TAG"; exit 1; }
|
||||
- name: fetch the repository and its tags
|
||||
# verify_release.py is run from the commit this workflow runs from: the
|
||||
# tag itself on a tag push, the default branch on a manual run. It
|
||||
# builds the tag's own source in a separate worktree.
|
||||
run: |
|
||||
git init -q .
|
||||
git remote add origin "$REPO_URL"
|
||||
git fetch -q origin "+refs/tags/*:refs/tags/*" "$GITHUB_SHA"
|
||||
git checkout -q --detach "$GITHUB_SHA"
|
||||
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
|
||||
git rev-parse -q --verify "refs/tags/$TAG^{commit}" >/dev/null \
|
||||
|| { echo "no tag $TAG in this repository"; exit 1; }
|
||||
- name: check the signed manifest and rebuild the .pyz from the tag
|
||||
run: |
|
||||
# The PyYAML release bastion_agent/build.py vendors, as a wheel only
|
||||
# and only with one of these sha256s (PyPI's, for Linux on x86_64 and
|
||||
# arm64, Python 3.12 and 3.13): the one package this job takes from
|
||||
# outside the repository, and this step runs before the upload step
|
||||
# that holds the token. A different PyYAML would only make the
|
||||
# rebuild differ and the step refuse.
|
||||
cat > "$RUNNER_TEMP/requirements.txt" <<'REQ'
|
||||
pyyaml==6.0.3 \
|
||||
--hash=sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc \
|
||||
--hash=sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28 \
|
||||
--hash=sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6 \
|
||||
--hash=sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c
|
||||
REQ
|
||||
/usr/bin/python3 -m venv /tmp/venv
|
||||
/tmp/venv/bin/pip install -q --disable-pip-version-check --only-binary :all: \
|
||||
--require-hashes -r "$RUNNER_TEMP/requirements.txt"
|
||||
/tmp/venv/bin/python -m bastion_agent.verify_release \
|
||||
--expect-version "$TAG" --out "$RUNNER_TEMP/release"
|
||||
ls -l "$RUNNER_TEMP/release"
|
||||
- name: the release notes, from that version's CHANGELOG section
|
||||
run: |
|
||||
/usr/bin/git show "$TAG:CHANGELOG.md" > "$RUNNER_TEMP/CHANGELOG.md"
|
||||
/usr/bin/python3 -I - "${TAG#v}" "$RUNNER_TEMP/CHANGELOG.md" > "$RUNNER_TEMP/notes.md" <<'PY'
|
||||
import re, sys
|
||||
ver, path = sys.argv[1], sys.argv[2]
|
||||
text = open(path, encoding="utf-8").read()
|
||||
m = re.search(r"^## %s[ \t]*\n(.*?)(?=^## |\Z)" % re.escape(ver), text, re.M | re.S)
|
||||
if not m:
|
||||
sys.exit("CHANGELOG.md at the tag has no section for %s" % ver)
|
||||
print(m.group(1).strip())
|
||||
print()
|
||||
print("---")
|
||||
print("The files below are this tag rebuilt by this repository's release workflow, "
|
||||
"attached only after their sha256 matched the signed manifest "
|
||||
"(`version.json` and its `.sig`). To check them yourself, run "
|
||||
"`python3 -m bastion_agent.verify_release` from a clone: "
|
||||
"see [Verify a release](https://github.com/%s#verify-a-release)."
|
||||
% "Privatae/bastion-agent")
|
||||
PY
|
||||
- name: attach the files to the release
|
||||
# The one step with the token. Its shell and Python are named by path,
|
||||
# and Python runs isolated (-I), so nothing an earlier step put on PATH
|
||||
# or in the environment chooses what runs here.
|
||||
shell: /usr/bin/bash -euo pipefail {0}
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GH_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
/usr/bin/python3 -I - "$RUNNER_TEMP/release" "$RUNNER_TEMP/notes.md" <<'PY'
|
||||
import json, os, sys, urllib.error, urllib.parse, urllib.request
|
||||
|
||||
out_dir, notes = sys.argv[1], sys.argv[2]
|
||||
api = os.environ.get("GITHUB_API_URL", "https://api.github.com").rstrip("/")
|
||||
repo, tag, token = os.environ["GH_REPO"], os.environ["TAG"], os.environ["GH_TOKEN"]
|
||||
|
||||
def call(method, url, body=None, ctype="application/json"):
|
||||
data = body if isinstance(body, bytes) or body is None else json.dumps(body).encode()
|
||||
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||
"Authorization": "Bearer " + token, "Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28", "Content-Type": ctype,
|
||||
"User-Agent": "bastion-agent-release"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=120) as r:
|
||||
raw = r.read()
|
||||
return r.status, json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, None
|
||||
|
||||
st, rel = call("GET", "%s/repos/%s/releases/tags/%s" % (api, repo, tag))
|
||||
if st == 404:
|
||||
body = open(notes, encoding="utf-8").read()
|
||||
st, rel = call("POST", "%s/repos/%s/releases" % (api, repo), {
|
||||
"tag_name": tag, "name": "bastion agent " + tag[1:], "body": body,
|
||||
"draft": False, "prerelease": False})
|
||||
if st != 201:
|
||||
sys.exit("creating the release for %s failed: HTTP %d" % (tag, st))
|
||||
print("created the release for %s" % tag)
|
||||
elif st != 200:
|
||||
sys.exit("reading the release for %s failed: HTTP %d" % (tag, st))
|
||||
else:
|
||||
print("the release for %s exists; its text is left as it is" % tag)
|
||||
|
||||
st, assets = call("GET", "%s/repos/%s/releases/%d/assets?per_page=100" % (api, repo, rel["id"]))
|
||||
if st != 200:
|
||||
sys.exit("listing the release's files failed: HTTP %d" % st)
|
||||
have = {a["name"]: a["id"] for a in assets}
|
||||
upload = rel["upload_url"].split("{", 1)[0]
|
||||
for name in sorted(os.listdir(out_dir)):
|
||||
if name in have:
|
||||
st, _ = call("DELETE", "%s/repos/%s/releases/assets/%d" % (api, repo, have[name]))
|
||||
if st != 204:
|
||||
sys.exit("replacing %s failed: HTTP %d" % (name, st))
|
||||
with open(os.path.join(out_dir, name), "rb") as fh:
|
||||
data = fh.read()
|
||||
st, a = call("POST", upload + "?" + urllib.parse.urlencode({"name": name}), data,
|
||||
"application/octet-stream")
|
||||
if st != 201:
|
||||
sys.exit("uploading %s failed: HTTP %d" % (name, st))
|
||||
print("attached %s (%d bytes)" % (a["name"], a["size"]))
|
||||
PY
|
||||
Reference in New Issue
Block a user