feat: add verified and resumable Gitea release synchronization

.gitea/workflows/sync-releases.yml

@@ -0,0 +1,57 @@
+name: Daily Release Archive
+
+on:
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: 'Only inspect and report; do not write releases'
+        default: 'true'
+        required: true
+      tag:
+        description: 'Optional source tag; empty means all published releases'
+        default: ''
+        required: false
+      verify_all:
+        description: 'Full verification (also performed by default)'
+        default: 'true'
+        required: false
+
+concurrency:
+  group: daily-release-archive
+  cancel-in-progress: false
+
+permissions:
+  contents: write
+
+jobs:
+  sync:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout trusted synchronization code
+        uses: https://github.com/actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          persist-credentials: false
+      - name: Test synchronization logic
+        run: python3 -m unittest discover -s tests -v
+      - name: Synchronize and verify releases
+        env:
+          SOURCE_GITEA_TOKEN: ${{ secrets.SOURCE_GITEA_TOKEN }}
+          TARGET_GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+          ARCHIVE_ANCHOR_SHA: '92634afb073b81216388c9288cbf275ba1e11a2c'
+          SYNC_DRY_RUN: ${{ github.event.inputs.dry_run }}
+          SYNC_TAG: ${{ github.event.inputs.tag }}
+        run: |
+          set -eu
+          args=()
+          if [ "${SYNC_DRY_RUN:-false}" = "true" ]; then args+=(--dry-run); fi
+          if [ -n "${SYNC_TAG:-}" ]; then args+=(--tag "$SYNC_TAG"); fi
+          python3 scripts/sync_releases.py "${args[@]}"
+      - name: Save synchronization report
+        if: always()
+        uses: https://github.com/actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: release-sync-report
+          path: sync-report.json
+          if-no-files-found: warn
+          retention-days: 90

.gitignore

@@ -0,0 +1,4 @@
+__pycache__/
+*.pyc
+sync-report.json
+*.token

scripts/sync_releases.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python3
+"""Archive Gitea releases. Standard library only; never executes source content."""
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import tempfile
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import zipfile
+
+MAX_BYTES = 512 * 1024 * 1024
+MARKER = '<!-- release-sync: git.bunnyruihan.com/ruihan/yidaconnector -->'
+
+
+class SyncError(Exception):
+    pass
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        return None
+
+
+class API:
+    def __init__(self, base, repo, token):
+        self.base, self.repo, self.token = base.rstrip('/'), repo, token
+        self.opener = urllib.request.build_opener(NoRedirect)
+
+    def request(self, method, path, body=None, binary=False, content_type=None):
+        url = self.base + '/api/v1/repos/' + self.repo + path
+        payload = body if binary else (json.dumps(body).encode() if body is not None else None)
+        headers = {'Authorization': 'token ' + self.token, 'Accept': 'application/json'}
+        if payload is not None:
+            headers['Content-Type'] = content_type or 'application/json'
+        for attempt in range(3):
+            try:
+                with self.opener.open(urllib.request.Request(url, data=payload, headers=headers, method=method), timeout=60) as response:
+                    data = response.read()
+                return json.loads(data) if data else None
+            except urllib.error.HTTPError as e:
+                if e.code == 404 and method == 'GET':
+                    return None
+                if method == 'GET' and e.code in (429, 500, 502, 503, 504) and attempt < 2:
+                    time.sleep(2 ** attempt)
+                    continue
+                raise SyncError(f'{method} {path.split("?")[0]} HTTP {e.code}') from None
+            except (urllib.error.URLError, TimeoutError, OSError):
+                if method == 'GET' and attempt < 2:
+                    time.sleep(2 ** attempt)
+                    continue
+                raise SyncError(f'{method} request failed; retry the workflow to reconcile state') from None
+
+    def pages(self, path):
+        result = []
+        for page in range(1, 10001):
+            data = self.request('GET', path + ('&' if '?' in path else '?') + f'limit=50&page={page}')
+            if not isinstance(data, list):
+                raise SyncError('Expected paginated list: ' + path)
+            if not data:
+                return result
+            result.extend(data)
+        raise SyncError('Pagination limit exceeded')
+
+    def download(self, asset, destination):
+        url = asset['browser_download_url']
+        origin = urllib.parse.urlsplit(self.base)
+        for attempt in range(3):
+            current = url
+            try:
+                for _ in range(5):
+                    parsed = urllib.parse.urlsplit(current)
+                    if parsed.scheme != 'https' or parsed.netloc != origin.netloc or parsed.username:
+                        raise SyncError('Attachment URL or redirect is outside the approved HTTPS origin')
+                    req = urllib.request.Request(current, headers={'Authorization': 'token ' + self.token})
+                    try:
+                        with self.opener.open(req, timeout=60) as response, open(destination, 'wb') as out:
+                            count = 0
+                            while True:
+                                chunk = response.read(1024 * 1024)
+                                if not chunk:
+                                    break
+                                count += len(chunk)
+                                if count > MAX_BYTES:
+                                    raise SyncError('Attachment exceeds 512 MiB limit')
+                                out.write(chunk)
+                        validate_file(destination, asset)
+                        return digest(destination)
+                    except urllib.error.HTTPError as e:
+                        if e.code in (301, 302, 303, 307, 308) and e.headers.get('Location'):
+                            current = urllib.parse.urljoin(current, e.headers['Location'])
+                            continue
+                        if e.code in (401, 403, 404):
+                            raise SyncError(f'Attachment download HTTP {e.code}') from None
+                        raise
+                raise SyncError('Too many attachment redirects')
+            except (urllib.error.URLError, TimeoutError, OSError):
+                if attempt == 2:
+                    raise SyncError('Attachment download failed after retries') from None
+                time.sleep(2 ** attempt)
+
+
+def digest(path):
+    h = hashlib.sha256()
+    with open(path, 'rb') as f:
+        for block in iter(lambda: f.read(1024 * 1024), b''):
+            h.update(block)
+    return h.hexdigest()
+
+
+def validate_file(path, asset):
+    if Path(path).stat().st_size != asset['size']:
+        raise SyncError('Attachment size mismatch: ' + asset['name'])
+    if asset['name'].lower().endswith('.zip'):
+        try:
+            with zipfile.ZipFile(path) as z:
+                if sum(i.file_size for i in z.infolist()) > 2 * 1024 ** 3:
+                    raise SyncError('ZIP expanded size exceeds verification limit')
+                if z.testzip() is not None:
+                    raise SyncError('ZIP CRC mismatch')
+        except (zipfile.BadZipFile, RuntimeError, NotImplementedError):
+            raise SyncError('Invalid or unsupported ZIP: ' + asset['name']) from None
+
+
+def validate_names(assets):
+    names = [a['name'] for a in assets]
+    if len(names) != len(set(names)):
+        raise SyncError('Source has duplicate attachment names')
+    for a in assets:
+        if not a['name'] or '/' in a['name'] or '\\' in a['name'] or a['name'].startswith(('.sync-', '.backup-')):
+            raise SyncError('Unsafe or reserved attachment name')
+        if a['size'] < 0 or a['size'] > MAX_BYTES:
+            raise SyncError('Attachment size outside configured limit')
+
+
+def release_body(source, source_url, hashes, source_sha):
+    notes = source.get('body') or ''
+    return notes + '\n\n---\n' + MARKER + '\n' + (
+        '本仓库仅归档发布附件；自动生成的 Source Code 压缩包不是原项目源码。\n\n'
+        f'来源：{source_url}\n\n源标签对应提交：`{source_sha}`\n\n'
+        f'原站发布时间：{source.get("published_at") or source.get("created_at") or "unknown"}\n\n'
+        '已校验附件 SHA-256：\n\n```text\n' +
+        '\n'.join(h + '  ' + name for name, h in sorted(hashes.items())) + '\n```\n'
+    )
+
+
+def sync_one(src, dst, source, anchor, dry_run, directory):
+    tag = source['tag_name']
+    key = urllib.parse.quote(tag, safe='')
+    source_assets = src.pages(f'/releases/{source["id"]}/assets')
+    validate_names(source_assets)
+    source_tag = src.request('GET', '/tags/' + key)
+    if not source_tag or not source_tag.get('commit', {}).get('sha'):
+        raise SyncError('Source tag commit cannot be resolved')
+    source_sha = source_tag['commit']['sha']
+    hashes, files = {}, {}
+    for a in source_assets:
+        p = Path(directory) / ('source-' + str(a['id']))
+        hashes[a['name']] = src.download(a, p)
+        files[a['name']] = p
+    fresh = src.pages(f'/releases/{source["id"]}/assets')
+    identity = lambda rows: sorted((a['id'], a['name'], a['size']) for a in rows)
+    if identity(fresh) != identity(source_assets):
+        raise SyncError('Source attachment list changed during download; retry later')
+    target = dst.request('GET', '/releases/tags/' + key)
+    source_url = src.base + '/' + src.repo + '/releases/tag/' + key
+    body = release_body(source, source_url, hashes, source_sha)
+    desired = {'name': source.get('name') or tag, 'body': body, 'prerelease': source.get('prerelease', False)}
+    if target and MARKER not in (target.get('body') or ''):
+        raise SyncError('Target release exists but is not managed by this synchronizer')
+    result = {'tag': tag, 'source_sha': source_sha, 'created': target is None, 'assets': [], 'metadata_changed': False}
+    if target is None and not dry_run:
+        target = dst.request('POST', '/releases', dict(desired, tag_name=tag, target_commitish=anchor, draft=True))
+    assets = dst.pages(f'/releases/{target["id"]}/assets') if target else []
+    for source_asset in source_assets:
+        name, sha = source_asset['name'], hashes[source_asset['name']]
+        existing = next((a for a in assets if a['name'] == name), None)
+        old_sha = None
+        if existing:
+            old_sha = dst.download(existing, Path(directory) / ('target-' + str(existing['id'])))
+        if old_sha == sha:
+            result['assets'].append({'name': name, 'sha256': sha, 'action': 'unchanged'})
+            continue
+        action = 'replace' if existing else 'add'
+        if not dry_run:
+            rid = target['id']
+            # Recover an upload accepted by the server before a previous connection failed.
+            temp_name = '.sync-' + hashlib.sha256((name + sha).encode()).hexdigest()[:24]
+            staging = next((a for a in assets if a['name'] == temp_name), None)
+            if staging is None:
+                staging = dst.request('POST', f'/releases/{rid}/assets?name=' + urllib.parse.quote(temp_name, safe=''),
+                                      files[name].read_bytes(), binary=True, content_type='application/octet-stream')
+                assets.append(staging)
+            staging_sha = dst.download(staging, Path(directory) / ('staged-' + str(staging['id'])))
+            if staging_sha != sha:
+                raise SyncError('Uploaded attachment SHA-256 mismatch')
+            if existing:
+                backup_name = '.backup-' + str(existing['id']) + '-' + name
+                dst.request('PATCH', f'/releases/{rid}/assets/{existing["id"]}', {'name': backup_name})
+                existing['name'] = backup_name
+            dst.request('PATCH', f'/releases/{rid}/assets/{staging["id"]}', {'name': name})
+            staging['name'] = name
+        result['assets'].append({'name': name, 'sha256': sha, 'action': action})
+    if target:
+        changed = target.get('draft', False) or any(target.get(k) != v for k, v in desired.items())
+        result['metadata_changed'] = changed
+        if changed and not dry_run:
+            dst.request('PATCH', f'/releases/{target["id"]}', dict(desired, draft=False))
+    else:
+        result['metadata_changed'] = True
+    return result
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--dry-run', action='store_true')
+    p.add_argument('--tag', default='')
+    p.add_argument('--verify-all', action='store_true', help='All attachments are always fully verified')
+    p.add_argument('--report', default='sync-report.json')
+    args = p.parse_args()
+    report = {'dry_run': args.dry_run, 'started_at': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'releases': [], 'errors': []}
+    try:
+        src = API('https://git.bunnyruihan.com:16666', 'ruihan/yidaconnector', os.environ['SOURCE_GITEA_TOKEN'])
+        dst = API('https://git.jskdjs.cn', 'ruihan/yidaconnector', os.environ['TARGET_GITEA_TOKEN'])
+        anchor = os.environ['ARCHIVE_ANCHOR_SHA']
+        if not re.fullmatch('[0-9a-f]{40}', anchor):
+            raise SyncError('Invalid archive anchor SHA')
+        info = dst.request('GET', '')
+        if not info or info['full_name'] != dst.repo or not info['private']:
+            raise SyncError('Expected private target archive repository')
+        releases = [r for r in src.pages('/releases') if not r.get('draft')]
+        if args.tag:
+            releases = [r for r in releases if r['tag_name'] == args.tag]
+            if not releases:
+                raise SyncError('Requested published source tag not found')
+        for source in reversed(releases):
+            try:
+                with tempfile.TemporaryDirectory(prefix='release-sync-') as directory:
+                    result = sync_one(src, dst, source, anchor, args.dry_run, directory)
+                report['releases'].append(result)
+                print(json.dumps(result, ensure_ascii=False), flush=True)
+            except Exception as e:
+                # Do not render untrusted response bodies, request headers or tokens.
+                message = str(e) if isinstance(e, SyncError) else type(e).__name__
+                report['errors'].append({'tag': source['tag_name'], 'error': message})
+                print('FAILED ' + source['tag_name'] + ': ' + message, flush=True)
+    except Exception as e:
+        report['errors'].append({'error': str(e) if isinstance(e, SyncError) else type(e).__name__})
+    finally:
+        report['finished_at'] = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
+        Path(args.report).write_text(json.dumps(report, ensure_ascii=False, indent=2))
+        print(f'SUMMARY releases={len(report["releases"])} errors={len(report["errors"])} dry_run={args.dry_run}')
+    return 1 if report['errors'] else 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

tests/test_sync.py

@@ -0,0 +1,123 @@
+import copy
+import io
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
+from sync_releases import API, MARKER, SyncError, digest, sync_one, validate_file, validate_names
+
+
+def zip_bytes(text):
+    b = io.BytesIO()
+    with zipfile.ZipFile(b, 'w') as z:
+        z.writestr('README.txt', text)
+    return b.getvalue()
+
+
+class Fake:
+    base = 'https://source.invalid'
+    repo = 'owner/repo'
+    def __init__(self, source=False):
+        self.assets = []
+        self.data = {}
+        self.release = {'id': 1, 'tag_name': 'v1', 'name': 'one', 'body': 'notes', 'draft': False} if source else None
+        self.writes = []
+        self.fail_rename = False
+    def add(self, name, content):
+        i = len(self.data) + 1
+        a = {'id': i, 'name': name, 'size': len(content)}
+        self.assets.append(a)
+        self.data[i] = content
+        return copy.deepcopy(a)
+    def pages(self, path):
+        return copy.deepcopy(self.assets)
+    def download(self, a, p):
+        Path(p).write_bytes(self.data[a['id']])
+        validate_file(p, a)
+        return digest(p)
+    def request(self, method, path, body=None, **kwargs):
+        if method == 'GET':
+            return {'commit': {'sha': 'a' * 40}} if path.startswith('/tags/') else copy.deepcopy(self.release)
+        self.writes.append((method, path))
+        if method == 'POST' and path == '/releases':
+            self.release = dict(body, id=2)
+            return copy.deepcopy(self.release)
+        if method == 'POST':
+            from urllib.parse import unquote
+            return self.add(unquote(path.split('name=')[1]), body)
+        if '/assets/' in path:
+            if self.fail_rename and body['name'] == 'app.zip':
+                self.fail_rename = False
+                raise SyncError('simulated interrupted rename')
+            item = next(a for a in self.assets if a['id'] == int(path.rsplit('/', 1)[1]))
+            item.update(body)
+            return copy.deepcopy(item)
+        self.release.update(body)
+        return copy.deepcopy(self.release)
+
+
+class Tests(unittest.TestCase):
+    def setUp(self):
+        self.src, self.dst = Fake(True), Fake()
+        self.src.add('app.zip', zip_bytes('version 1'))
+    def run_sync(self, dry=False):
+        with tempfile.TemporaryDirectory() as d:
+            return sync_one(self.src, self.dst, self.src.release, 'a' * 40, dry, d)
+    def test_dry_run_writes_nothing(self):
+        self.run_sync(True)
+        self.assertEqual([], self.dst.writes)
+    def test_new_release_published_only_after_verified_assets(self):
+        self.run_sync()
+        self.assertFalse(self.dst.release['draft'])
+        self.assertEqual('app.zip', self.dst.assets[0]['name'])
+        self.assertEqual(('PATCH', '/releases/2'), self.dst.writes[-1])
+    def test_repeat_is_noop(self):
+        self.run_sync()
+        self.dst.writes.clear()
+        result = self.run_sync()
+        self.assertEqual([], self.dst.writes)
+        self.assertEqual('unchanged', result['assets'][0]['action'])
+    def test_same_name_same_size_different_bytes_is_detected(self):
+        self.run_sync()
+        self.src.data[1] = zip_bytes('version 2')
+        result = self.run_sync()
+        self.assertEqual('replace', result['assets'][0]['action'])
+        self.assertTrue(any(a['name'].startswith('.backup-') for a in self.dst.assets))
+    def test_interrupted_swap_recovers(self):
+        self.run_sync()
+        self.src.data[1] = zip_bytes('version 2')
+        self.dst.fail_rename = True
+        with self.assertRaises(SyncError): self.run_sync()
+        self.run_sync()
+        self.assertEqual(1, sum(a['name'] == 'app.zip' for a in self.dst.assets))
+        self.assertEqual(2, len(self.dst.assets))
+    def test_unmanaged_release_never_overwritten(self):
+        self.dst.release = dict(self.src.release)
+        with self.assertRaises(SyncError): self.run_sync()
+        self.assertEqual([], self.dst.writes)
+    def test_deleted_source_asset_retained(self):
+        self.run_sync()
+        self.src.assets.clear()
+        self.run_sync()
+        self.assertEqual(1, len(self.dst.assets))
+    def test_html_error_not_zip(self):
+        with tempfile.TemporaryDirectory() as d:
+            p = Path(d) / 'bad'; p.write_bytes(b'Not found.\n')
+            with self.assertRaises(SyncError): validate_file(p, {'size': 11, 'name': 'bad.zip'})
+    def test_unsafe_names_rejected(self):
+        for name in ['../secret', '.sync-x', '.backup-x', 'a\\b']:
+            with self.assertRaises(SyncError): validate_names([{'name': name, 'size': 1}])
+    def test_pagination_does_not_assume_server_page_size(self):
+        api = API('https://example.invalid', 'a/b', '')
+        pages = iter([[1, 2], [3], []])
+        api.request = lambda *a: next(pages)
+        self.assertEqual([1, 2, 3], api.pages('/releases'))
+    def test_cross_origin_download_rejected_before_request(self):
+        api = API('https://example.invalid', 'a/b', 'secret')
+        with self.assertRaises(SyncError):
+            api.download({'browser_download_url': 'https://other.invalid/file'}, '/tmp/unused')
+
+
+if __name__ == '__main__': unittest.main()