ci(parser): automate upstream parser sync
This commit is contained in:
Aethersailor
1
.github/upstream-subconverter.applied.json
vendored
Normal file
1
.github/upstream-subconverter.applied.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
1
.github/upstream-subconverter.seen
vendored
Normal file
1
.github/upstream-subconverter.seen
vendored
Normal file
@@ -0,0 +1 @@
|
||||
633ecd5a3b33cf288658f0910fb2cc5faabd351c
|
||||
1
.github/upstream-subconverter.skipped.json
vendored
Normal file
1
.github/upstream-subconverter.skipped.json
vendored
Normal file
@@ -0,0 +1 @@
|
||||
[]
|
||||
309
.github/workflows/sync-upstream-parser.yml
vendored
Normal file
309
.github/workflows/sync-upstream-parser.yml
vendored
Normal file
@@ -0,0 +1,309 @@
|
||||
name: Sync Upstream Parser
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 20 * * *"
|
||||
workflow_dispatch: {}
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
issues: write
|
||||
|
||||
concurrency:
|
||||
group: sync-upstream-parser-dev
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
# Scheduled workflows are read from the default branch, but parser syncs
|
||||
# must be planned, tested, and committed on dev.
|
||||
TARGET_BRANCH: dev
|
||||
UPSTREAM_REPO: https://github.com/asdlokj1qpi233/subconverter.git
|
||||
UPSTREAM_BRANCH: master
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout dev
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ env.TARGET_BRANCH }}
|
||||
fetch-depth: 0
|
||||
token: ${{ secrets.PAT_TOKEN || github.token }}
|
||||
|
||||
- name: Configure Git
|
||||
run: |
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
|
||||
- name: Fetch upstream
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git remote add upstream "$UPSTREAM_REPO" 2>/dev/null || git remote set-url upstream "$UPSTREAM_REPO"
|
||||
git fetch --no-tags upstream "$UPSTREAM_BRANCH"
|
||||
|
||||
- name: Check protected integrations before sync
|
||||
run: python3 scripts/check_sync_guards.py
|
||||
|
||||
- name: Plan upstream parser sync
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/sync_upstream_parser.py plan \
|
||||
--upstream-ref "upstream/$UPSTREAM_BRANCH" \
|
||||
--max-commits 30 \
|
||||
--output upstream-sync-candidates.json \
|
||||
--report upstream-sync-plan.md
|
||||
|
||||
- name: Inspect sync plan
|
||||
id: plan
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY' >> "$GITHUB_OUTPUT"
|
||||
import json
|
||||
data = json.load(open("upstream-sync-candidates.json", encoding="utf-8"))
|
||||
candidates = data.get("candidates", [])
|
||||
reviewable = [item for item in candidates if item.get("safe_by_rules")]
|
||||
print(f"candidate_count={len(candidates)}")
|
||||
print(f"reviewable_count={len(reviewable)}")
|
||||
print(f"has_candidates={'true' if candidates else 'false'}")
|
||||
print(f"has_reviewable={'true' if reviewable else 'false'}")
|
||||
PY
|
||||
|
||||
- name: Check Copilot token
|
||||
id: copilot_token
|
||||
if: steps.plan.outputs.has_reviewable == 'true'
|
||||
env:
|
||||
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
|
||||
run: |
|
||||
if [ -n "${COPILOT_GITHUB_TOKEN:-}" ]; then
|
||||
echo "available=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "available=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Set up Node.js
|
||||
if: steps.plan.outputs.has_reviewable == 'true' && steps.copilot_token.outputs.available == 'true'
|
||||
uses: actions/setup-node@v6
|
||||
|
||||
- name: Ask Copilot to classify parser candidates
|
||||
id: copilot_classify
|
||||
if: steps.plan.outputs.has_reviewable == 'true' && steps.copilot_token.outputs.available == 'true'
|
||||
continue-on-error: true
|
||||
env:
|
||||
COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
npm install -g @github/copilot
|
||||
|
||||
PROMPT="$(cat <<'EOF'
|
||||
You are reviewing upstream parser changes for SubConverter-Extended.
|
||||
|
||||
The project has protected Mihomo/proxy-provider integrations. A change
|
||||
may be automatically applied only when it is a low-risk parser-only
|
||||
update and does not require output adapter changes.
|
||||
|
||||
Return only strict JSON. Do not use Markdown.
|
||||
|
||||
Required schema:
|
||||
{
|
||||
"decisions": [
|
||||
{
|
||||
"sha": "full commit sha from the plan",
|
||||
"decision": "safe_parser_only | needs_output_adapter | touches_protected_area | unsafe | skip",
|
||||
"risk": "low | medium | high",
|
||||
"reason": "short explanation",
|
||||
"required_tests": ["test names"]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Only use "safe_parser_only" with "low" risk when the diff updates
|
||||
parser-only behavior and can be safely applied without touching
|
||||
Mihomo parser bridge, RawParams pass-through, proxy-provider logic,
|
||||
FetchContext handling, nodemanip.cpp, or Clash output internals.
|
||||
EOF
|
||||
)"
|
||||
|
||||
PROMPT="$PROMPT
|
||||
|
||||
$(cat upstream-sync-candidates.json)"
|
||||
|
||||
copilot -p "$PROMPT" --no-ask-user > upstream-sync-decisions.raw
|
||||
|
||||
python3 - <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
raw = Path("upstream-sync-decisions.raw").read_text(encoding="utf-8")
|
||||
start = raw.find("{")
|
||||
end = raw.rfind("}")
|
||||
if start == -1 or end == -1 or end < start:
|
||||
raise SystemExit("Copilot did not return a JSON object.")
|
||||
data = json.loads(raw[start:end + 1])
|
||||
if "decisions" not in data or not isinstance(data["decisions"], list):
|
||||
raise SystemExit("Copilot JSON is missing decisions array.")
|
||||
Path("upstream-sync-decisions.json").write_text(
|
||||
json.dumps(data, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
|
||||
- name: Create empty decisions for non-reviewable plan
|
||||
if: steps.plan.outputs.has_candidates == 'true' && steps.plan.outputs.has_reviewable != 'true'
|
||||
run: printf '%s\n' '{"decisions":[]}' > upstream-sync-decisions.json
|
||||
|
||||
- name: Apply approved upstream parser updates
|
||||
id: apply
|
||||
if: steps.plan.outputs.has_candidates == 'true' && (steps.plan.outputs.has_reviewable != 'true' || steps.copilot_classify.outcome == 'success')
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/sync_upstream_parser.py apply \
|
||||
--plan upstream-sync-candidates.json \
|
||||
--decisions upstream-sync-decisions.json \
|
||||
--result upstream-sync-result.json \
|
||||
--report upstream-sync-result.md
|
||||
|
||||
python3 - <<'PY' >> "$GITHUB_OUTPUT"
|
||||
import json
|
||||
data = json.load(open("upstream-sync-result.json", encoding="utf-8"))
|
||||
print(f"applied_count={len(data.get('applied', []))}")
|
||||
print(f"skipped_count={len(data.get('skipped', []))}")
|
||||
print(f"has_applied={'true' if data.get('applied') else 'false'}")
|
||||
print(f"has_skipped={'true' if data.get('skipped') else 'false'}")
|
||||
PY
|
||||
|
||||
- name: Check protected integrations after sync
|
||||
if: steps.apply.outcome == 'success'
|
||||
run: python3 scripts/check_sync_guards.py
|
||||
|
||||
- name: Determine changed source files
|
||||
id: changes
|
||||
if: steps.apply.outcome == 'success'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet; then
|
||||
echo "has_changes=false" >> "$GITHUB_OUTPUT"
|
||||
echo "has_source_changes=false" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "has_changes=true" >> "$GITHUB_OUTPUT"
|
||||
if git diff --name-only | grep -Eq '^(src|bridge|include|CMakeLists\.txt|Dockerfile)'; then
|
||||
echo "has_source_changes=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "has_source_changes=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and smoke test source changes
|
||||
if: steps.changes.outputs.has_source_changes == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker build --target builder -t subconverter-upstream-sync-test --build-arg THREADS=2 .
|
||||
|
||||
CID="$(docker run -d -e PORT=25500 -p '127.0.0.1::25500' subconverter-upstream-sync-test /src/subconverter -f /src/base/pref.example.toml)"
|
||||
trap 'docker rm -f "$CID" >/dev/null 2>&1 || true' EXIT
|
||||
|
||||
HOST_PORT="$(docker port "$CID" 25500/tcp | awk -F: 'END {print $NF}')"
|
||||
BASE_URL="http://127.0.0.1:${HOST_PORT}"
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if curl -fsS "$BASE_URL/version" >/tmp/version.out; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
curl -fsS "$BASE_URL/version" >/tmp/version.out
|
||||
|
||||
curl -fsS "$BASE_URL/sub?target=clash&url=https%3A%2F%2Fexample.com%2Fsub" > /tmp/clash-provider.yml
|
||||
grep -q "proxy-providers:" /tmp/clash-provider.yml
|
||||
|
||||
curl -fsS "$BASE_URL/sub?target=clash&url=ss%3A%2F%2FY2hhY2hhMjAtaWV0Zi1wb2x5MTMwNTpwYXNzd29yZA%40www.example.com%3A1080%23Example" > /tmp/clash-node.yml
|
||||
grep -Eq "type: ss|type: \"ss\"" /tmp/clash-node.yml
|
||||
|
||||
curl -fsS "$BASE_URL/sub?target=singbox&url=ss%3A%2F%2FY2hhY2hhMjAtaWV0Zi1wb2x5MTMwNTpwYXNzd29yZA%40www.example.com%3A1080%23Example" > /tmp/singbox.json
|
||||
python3 -m json.tool /tmp/singbox.json >/dev/null
|
||||
|
||||
curl -fsS "$BASE_URL/sub?target=surge&url=ss%3A%2F%2FY2hhY2hhMjAtaWV0Zi1wb2x5MTMwNTpwYXNzd29yZA%40www.example.com%3A1080%23Example" > /tmp/surge.conf
|
||||
test -s /tmp/surge.conf
|
||||
|
||||
- name: Commit sync result to dev
|
||||
if: steps.changes.outputs.has_changes == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git add \
|
||||
.github/upstream-subconverter.seen \
|
||||
.github/upstream-subconverter.applied.json \
|
||||
.github/upstream-subconverter.skipped.json \
|
||||
src/parser/subparser.cpp \
|
||||
src/parser/subparser.h \
|
||||
src/parser/config/proxy.h
|
||||
|
||||
if git diff --staged --quiet; then
|
||||
echo "No staged sync changes."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "${{ steps.apply.outputs.has_applied }}" = "true" ]; then
|
||||
git commit -m "chore(parser): sync upstream parser updates"
|
||||
else
|
||||
git commit -m "chore(parser): update upstream sync marker [skip ci]"
|
||||
fi
|
||||
|
||||
git fetch --no-tags origin "refs/heads/$TARGET_BRANCH:refs/remotes/origin/$TARGET_BRANCH"
|
||||
git rebase "origin/$TARGET_BRANCH"
|
||||
git push origin "HEAD:refs/heads/$TARGET_BRANCH"
|
||||
|
||||
- name: Upload sync reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: upstream-parser-sync-report
|
||||
path: |
|
||||
upstream-sync-candidates.json
|
||||
upstream-sync-decisions.raw
|
||||
upstream-sync-decisions.json
|
||||
upstream-sync-plan.md
|
||||
upstream-sync-result.json
|
||||
upstream-sync-result.md
|
||||
if-no-files-found: ignore
|
||||
|
||||
- name: Report sync items that need attention
|
||||
if: always() && (steps.apply.outputs.has_skipped == 'true' || (steps.plan.outputs.has_reviewable == 'true' && (steps.copilot_token.outputs.available != 'true' || steps.copilot_classify.outcome != 'success')))
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const title = 'Upstream parser sync requires attention';
|
||||
const marker = '<!-- upstream-parser-sync-attention -->';
|
||||
let body = `${marker}\nAutomated upstream parser sync on \`${process.env.TARGET_BRANCH}\` needs attention.\n\n`;
|
||||
|
||||
if (fs.existsSync('upstream-sync-result.md')) {
|
||||
body += fs.readFileSync('upstream-sync-result.md', 'utf8');
|
||||
} else if (fs.existsSync('upstream-sync-plan.md')) {
|
||||
body += fs.readFileSync('upstream-sync-plan.md', 'utf8');
|
||||
body += '\n\nCopilot classification was unavailable or failed, so reviewable candidates were not applied.\n';
|
||||
}
|
||||
|
||||
const {data: issues} = await github.rest.issues.listForRepo({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
state: 'open',
|
||||
labels: undefined,
|
||||
per_page: 50
|
||||
});
|
||||
const existing = issues.find(issue => issue.title === title);
|
||||
if (existing) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: existing.number,
|
||||
body
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title,
|
||||
body
|
||||
});
|
||||
}
|
||||
7
.gitignore
vendored
7
.gitignore
vendored
@@ -13,3 +13,10 @@ bridge/libmihomo.a
|
||||
bridge/libmihomo.so
|
||||
bridge/libmihomo.dll
|
||||
bridge/libmihomo.dll.a
|
||||
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
upstream-sync-*.json
|
||||
upstream-sync-*.md
|
||||
upstream-sync-decisions.raw
|
||||
|
||||
130
scripts/check_sync_guards.py
Normal file
130
scripts/check_sync_guards.py
Normal file
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Guard checks for automated upstream parser syncs.
|
||||
|
||||
The sync workflow may import parser fixes from upstream subconverter, but it
|
||||
must never erase this fork's Mihomo/proxy-provider integration. This script
|
||||
keeps those invariants machine-checkable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
CHECKS = [
|
||||
{
|
||||
"id": "mihomo-parser-compile-flag",
|
||||
"file": "src/generator/config/nodemanip.cpp",
|
||||
"pattern": r"USE_MIHOMO_PARSER",
|
||||
"message": "Mihomo parser compile flag is missing from nodemanip.cpp.",
|
||||
},
|
||||
{
|
||||
"id": "mihomo-parser-call",
|
||||
"file": "src/generator/config/nodemanip.cpp",
|
||||
"pattern": r"mihomo::parseSubscription",
|
||||
"message": "Mihomo parser call is missing from nodemanip.cpp.",
|
||||
},
|
||||
{
|
||||
"id": "fetch-context",
|
||||
"file": "src/generator/config/nodemanip.h",
|
||||
"pattern": r"FetchContext\s+fetch_context",
|
||||
"message": "FetchContext-aware parsing state is missing.",
|
||||
},
|
||||
{
|
||||
"id": "rawparams-field",
|
||||
"file": "src/parser/config/proxy.h",
|
||||
"pattern": r"std::map<\s*String\s*,\s*String\s*>\s+RawParams\s*;",
|
||||
"message": "Proxy.RawParams must be preserved for Mihomo pass-through.",
|
||||
},
|
||||
{
|
||||
"id": "rawparams-to-clash",
|
||||
"file": "src/generator/config/subexport.cpp",
|
||||
"pattern": r"if\s*\(!x\.RawParams\.empty\(\)\)",
|
||||
"message": "Clash RawParams pass-through block is missing.",
|
||||
},
|
||||
{
|
||||
"id": "proxy-provider-output",
|
||||
"file": "src/generator/config/subexport.cpp",
|
||||
"pattern": r"proxy-providers",
|
||||
"message": "proxy-providers output support is missing.",
|
||||
},
|
||||
{
|
||||
"id": "provider-direct-option",
|
||||
"file": "src/generator/config/subexport.h",
|
||||
"pattern": r"provider_proxy_direct",
|
||||
"message": "provider_proxy_direct option is missing.",
|
||||
},
|
||||
{
|
||||
"id": "provider-name-override",
|
||||
"file": "src/generator/config/subexport.cpp",
|
||||
"pattern": r"buildProviderProxyNameOverride",
|
||||
"message": "provider proxy-name override support is missing.",
|
||||
},
|
||||
{
|
||||
"id": "provider-creation",
|
||||
"file": "src/handler/interfaces.cpp",
|
||||
"pattern": r"ProxyProvider\s+provider",
|
||||
"message": "provider creation path is missing from request handling.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def run_checks() -> list[dict[str, str]]:
|
||||
failures: list[dict[str, str]] = []
|
||||
for check in CHECKS:
|
||||
path = ROOT / check["file"]
|
||||
if not path.exists():
|
||||
failures.append(
|
||||
{
|
||||
"id": check["id"],
|
||||
"file": check["file"],
|
||||
"message": f"Required file is missing: {check['file']}",
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
text = path.read_text(encoding="utf-8", errors="ignore")
|
||||
if not re.search(check["pattern"], text, flags=re.MULTILINE):
|
||||
failures.append(
|
||||
{
|
||||
"id": check["id"],
|
||||
"file": check["file"],
|
||||
"message": check["message"],
|
||||
}
|
||||
)
|
||||
|
||||
return failures
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--json", action="store_true", help="emit JSON")
|
||||
args = parser.parse_args()
|
||||
|
||||
failures = run_checks()
|
||||
result = {"ok": not failures, "failures": failures}
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
elif failures:
|
||||
print("Sync guard checks failed:", file=sys.stderr)
|
||||
for failure in failures:
|
||||
print(
|
||||
f"- {failure['id']} ({failure['file']}): {failure['message']}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
print("Sync guard checks passed.")
|
||||
|
||||
return 0 if not failures else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
493
scripts/sync_upstream_parser.py
Normal file
493
scripts/sync_upstream_parser.py
Normal file
@@ -0,0 +1,493 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plan and apply guarded parser syncs from upstream subconverter.
|
||||
|
||||
This script intentionally does not trust upstream diffs by default. It plans
|
||||
candidate commits, lets CI/Copilot classify them, applies only low-risk parser
|
||||
updates, and records skipped commits so one unsafe upstream commit does not
|
||||
block later safe ones.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
SEEN_FILE = ROOT / ".github" / "upstream-subconverter.seen"
|
||||
APPLIED_FILE = ROOT / ".github" / "upstream-subconverter.applied.json"
|
||||
SKIPPED_FILE = ROOT / ".github" / "upstream-subconverter.skipped.json"
|
||||
|
||||
ALLOWED_AUTO_PATHS = {
|
||||
"src/parser/subparser.cpp",
|
||||
"src/parser/subparser.h",
|
||||
"src/parser/config/proxy.h",
|
||||
}
|
||||
|
||||
REPORT_ONLY_PATHS = {
|
||||
"src/generator/config/subexport.cpp",
|
||||
}
|
||||
|
||||
PROTECTED_PATHS = {
|
||||
"src/generator/config/nodemanip.cpp",
|
||||
"src/generator/config/nodemanip.h",
|
||||
"src/generator/config/subexport.h",
|
||||
"src/parser/mihomo_bridge.cpp",
|
||||
"src/parser/mihomo_bridge.h",
|
||||
"src/parser/mihomo_schemes.h",
|
||||
"src/parser/param_compat.h",
|
||||
"bridge/converter.go",
|
||||
"bridge/go.mod",
|
||||
"bridge/go.sum",
|
||||
}
|
||||
|
||||
PROTECTED_PREFIXES = (
|
||||
"bridge/",
|
||||
)
|
||||
|
||||
SAFE_DECISION = "safe_parser_only"
|
||||
|
||||
|
||||
def git(*args: str, input_text: str | None = None, check: bool = True) -> str:
|
||||
proc = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=ROOT,
|
||||
input=input_text,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"git {' '.join(args)} failed with {proc.returncode}\n{proc.stderr}"
|
||||
)
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def run(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
list(args),
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def read_json_array(path: Path) -> list[dict[str, Any]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
data = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return []
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def write_json(path: Path, data: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def read_seen() -> str:
|
||||
if not SEEN_FILE.exists():
|
||||
return ""
|
||||
return SEEN_FILE.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def path_is_protected(path: str) -> bool:
|
||||
return path in PROTECTED_PATHS or any(path.startswith(prefix) for prefix in PROTECTED_PREFIXES)
|
||||
|
||||
|
||||
def commit_files(sha: str) -> list[str]:
|
||||
output = git("diff-tree", "--no-commit-id", "--name-only", "-r", sha)
|
||||
return [line.strip() for line in output.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def commit_subject(sha: str) -> str:
|
||||
return git("show", "-s", "--format=%s", sha).strip()
|
||||
|
||||
|
||||
def commit_patch(sha: str, paths: list[str], max_chars: int = 12000) -> str:
|
||||
if not paths:
|
||||
return ""
|
||||
patch = git("show", "--format=fuller", "--stat", "--patch", sha, "--", *paths)
|
||||
if len(patch) <= max_chars:
|
||||
return patch
|
||||
return patch[:max_chars] + "\n\n[diff truncated]\n"
|
||||
|
||||
|
||||
def classify_commit(sha: str) -> dict[str, Any]:
|
||||
files = commit_files(sha)
|
||||
allowed = [path for path in files if path in ALLOWED_AUTO_PATHS]
|
||||
protected = [path for path in files if path_is_protected(path)]
|
||||
report_only = [path for path in files if path in REPORT_ONLY_PATHS]
|
||||
other = [
|
||||
path
|
||||
for path in files
|
||||
if path not in ALLOWED_AUTO_PATHS
|
||||
and path not in REPORT_ONLY_PATHS
|
||||
and not path_is_protected(path)
|
||||
]
|
||||
|
||||
if not allowed:
|
||||
rule_decision = "ignore_no_parser_changes"
|
||||
safe_by_rules = False
|
||||
reason = "Commit does not change parser whitelist files."
|
||||
elif protected:
|
||||
rule_decision = "skip_protected_path"
|
||||
safe_by_rules = False
|
||||
reason = "Commit touches protected project-specific integration paths."
|
||||
elif report_only or other:
|
||||
rule_decision = "needs_human_or_ai_report"
|
||||
safe_by_rules = False
|
||||
reason = "Commit changes parser files plus non-whitelisted files."
|
||||
else:
|
||||
rule_decision = "candidate"
|
||||
safe_by_rules = True
|
||||
reason = "Commit changes only parser whitelist files."
|
||||
|
||||
return {
|
||||
"sha": sha,
|
||||
"short_sha": sha[:12],
|
||||
"subject": commit_subject(sha),
|
||||
"files": files,
|
||||
"allowed_paths": allowed,
|
||||
"protected_paths": protected,
|
||||
"report_only_paths": report_only,
|
||||
"other_paths": other,
|
||||
"safe_by_rules": safe_by_rules,
|
||||
"rule_decision": rule_decision,
|
||||
"reason": reason,
|
||||
"patch_excerpt": commit_patch(sha, allowed or files[:10]),
|
||||
}
|
||||
|
||||
|
||||
def plan(args: argparse.Namespace) -> int:
|
||||
upstream_head = git("rev-parse", args.upstream_ref).strip()
|
||||
seen = read_seen()
|
||||
bootstrap = False
|
||||
|
||||
if not seen:
|
||||
bootstrap = True
|
||||
commits: list[str] = []
|
||||
else:
|
||||
exists = run("git", "cat-file", "-e", f"{seen}^{{commit}}", check=False)
|
||||
if exists.returncode != 0:
|
||||
bootstrap = True
|
||||
commits = []
|
||||
else:
|
||||
commits_out = git(
|
||||
"rev-list",
|
||||
"--reverse",
|
||||
"--no-merges",
|
||||
f"{seen}..{args.upstream_ref}",
|
||||
)
|
||||
commits = [line.strip() for line in commits_out.splitlines() if line.strip()]
|
||||
|
||||
if args.max_commits > 0:
|
||||
commits = commits[: args.max_commits]
|
||||
|
||||
candidates = [classify_commit(sha) for sha in commits]
|
||||
data = {
|
||||
"generated_at": utc_now(),
|
||||
"upstream_ref": args.upstream_ref,
|
||||
"upstream_head": upstream_head,
|
||||
"seen": seen,
|
||||
"bootstrap": bootstrap,
|
||||
"allowed_auto_paths": sorted(ALLOWED_AUTO_PATHS),
|
||||
"protected_paths": sorted(PROTECTED_PATHS),
|
||||
"protected_prefixes": list(PROTECTED_PREFIXES),
|
||||
"safe_decision": SAFE_DECISION,
|
||||
"candidates": candidates,
|
||||
}
|
||||
|
||||
write_json(Path(args.output), data)
|
||||
write_plan_report(Path(args.report), data)
|
||||
|
||||
safe_count = sum(1 for item in candidates if item["safe_by_rules"])
|
||||
print(f"Planned {len(candidates)} upstream commits ({safe_count} rule-safe).")
|
||||
if bootstrap:
|
||||
print("No seen marker was available; plan bootstrapped without candidates.")
|
||||
return 0
|
||||
|
||||
|
||||
def write_plan_report(path: Path, data: dict[str, Any]) -> None:
|
||||
lines = [
|
||||
"# Upstream Parser Sync Plan",
|
||||
"",
|
||||
f"- Generated: {data['generated_at']}",
|
||||
f"- Upstream ref: `{data['upstream_ref']}`",
|
||||
f"- Seen: `{data['seen'] or 'none'}`",
|
||||
f"- Upstream head: `{data['upstream_head']}`",
|
||||
f"- Bootstrap: `{data['bootstrap']}`",
|
||||
"",
|
||||
"## Candidates",
|
||||
"",
|
||||
]
|
||||
if not data["candidates"]:
|
||||
lines.append("No candidate commits.")
|
||||
for item in data["candidates"]:
|
||||
lines.extend(
|
||||
[
|
||||
f"### `{item['short_sha']}` {item['subject']}",
|
||||
"",
|
||||
f"- Rule decision: `{item['rule_decision']}`",
|
||||
f"- Safe by rules: `{item['safe_by_rules']}`",
|
||||
f"- Reason: {item['reason']}",
|
||||
f"- Files: {', '.join(f'`{path}`' for path in item['files']) or 'none'}",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def load_decisions(path: Path) -> dict[str, dict[str, Any]]:
|
||||
text = path.read_text(encoding="utf-8").strip()
|
||||
data = json.loads(text)
|
||||
decisions = data.get("decisions", data)
|
||||
if not isinstance(decisions, list):
|
||||
raise ValueError("Copilot decision file must contain a decisions array.")
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for item in decisions:
|
||||
if not isinstance(item, dict) or "sha" not in item:
|
||||
continue
|
||||
result[item["sha"]] = item
|
||||
return result
|
||||
|
||||
|
||||
def snapshot_paths(paths: list[str]) -> dict[str, bytes | None]:
|
||||
snapshot: dict[str, bytes | None] = {}
|
||||
for path in paths:
|
||||
full = ROOT / path
|
||||
snapshot[path] = full.read_bytes() if full.exists() else None
|
||||
return snapshot
|
||||
|
||||
|
||||
def restore_snapshot(snapshot: dict[str, bytes | None]) -> None:
|
||||
for path, content in snapshot.items():
|
||||
full = ROOT / path
|
||||
if content is None:
|
||||
if full.exists():
|
||||
full.unlink()
|
||||
else:
|
||||
full.parent.mkdir(parents=True, exist_ok=True)
|
||||
full.write_bytes(content)
|
||||
run("git", "reset", "--", *snapshot.keys(), check=False)
|
||||
|
||||
|
||||
def apply_patch_for_commit(sha: str, paths: list[str]) -> tuple[bool, str]:
|
||||
patch = git("show", "--format=", "--binary", sha, "--", *paths)
|
||||
if not patch.strip():
|
||||
return False, "No patch content for allowed parser files."
|
||||
|
||||
check = subprocess.run(
|
||||
["git", "apply", "-3", "--check", "--whitespace=nowarn", "-"],
|
||||
cwd=ROOT,
|
||||
input=patch,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if check.returncode != 0:
|
||||
return False, check.stderr.strip() or "git apply --check failed."
|
||||
|
||||
applied = subprocess.run(
|
||||
["git", "apply", "-3", "--whitespace=nowarn", "-"],
|
||||
cwd=ROOT,
|
||||
input=patch,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
if applied.returncode != 0:
|
||||
return False, applied.stderr.strip() or "git apply failed."
|
||||
return True, "Applied."
|
||||
|
||||
|
||||
def run_guards() -> tuple[bool, str]:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, "scripts/check_sync_guards.py", "--json"],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
output = (proc.stdout or "") + (proc.stderr or "")
|
||||
return proc.returncode == 0, output.strip()
|
||||
|
||||
|
||||
def append_state(path: Path, entries: list[dict[str, Any]]) -> None:
|
||||
if not entries:
|
||||
return
|
||||
current = read_json_array(path)
|
||||
current.extend(entries)
|
||||
write_json(path, current)
|
||||
|
||||
|
||||
def apply(args: argparse.Namespace) -> int:
|
||||
plan_data = json.loads(Path(args.plan).read_text(encoding="utf-8"))
|
||||
decisions = load_decisions(Path(args.decisions))
|
||||
applied_entries: list[dict[str, Any]] = []
|
||||
skipped_entries: list[dict[str, Any]] = []
|
||||
ignored_entries: list[dict[str, Any]] = []
|
||||
|
||||
for item in plan_data.get("candidates", []):
|
||||
sha = item["sha"]
|
||||
decision = decisions.get(sha)
|
||||
record_base = {
|
||||
"sha": sha,
|
||||
"subject": item["subject"],
|
||||
"time": utc_now(),
|
||||
}
|
||||
|
||||
if item.get("rule_decision") == "ignore_no_parser_changes":
|
||||
ignored_entries.append(
|
||||
{**record_base, "reason": "No parser whitelist files changed."}
|
||||
)
|
||||
continue
|
||||
|
||||
if not item.get("safe_by_rules"):
|
||||
skipped_entries.append(
|
||||
{
|
||||
**record_base,
|
||||
"reason": item.get("reason", "Rejected by deterministic rules."),
|
||||
"rule_decision": item.get("rule_decision"),
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
if not decision:
|
||||
skipped_entries.append(
|
||||
{**record_base, "reason": "No Copilot decision was supplied."}
|
||||
)
|
||||
continue
|
||||
|
||||
if decision.get("decision") != SAFE_DECISION or decision.get("risk") != "low":
|
||||
skipped_entries.append(
|
||||
{
|
||||
**record_base,
|
||||
"reason": decision.get("reason", "Copilot did not approve automatic sync."),
|
||||
"copilot_decision": decision,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
paths = item.get("allowed_paths", [])
|
||||
backup = snapshot_paths(paths)
|
||||
ok, message = apply_patch_for_commit(sha, paths)
|
||||
if not ok:
|
||||
restore_snapshot(backup)
|
||||
skipped_entries.append({**record_base, "reason": message})
|
||||
continue
|
||||
|
||||
guards_ok, guards_output = run_guards()
|
||||
if not guards_ok:
|
||||
restore_snapshot(backup)
|
||||
skipped_entries.append(
|
||||
{
|
||||
**record_base,
|
||||
"reason": "Guard checks failed after applying patch.",
|
||||
"guard_output": guards_output,
|
||||
}
|
||||
)
|
||||
continue
|
||||
|
||||
applied_entries.append(
|
||||
{
|
||||
**record_base,
|
||||
"paths": paths,
|
||||
"copilot_reason": decision.get("reason", ""),
|
||||
}
|
||||
)
|
||||
|
||||
append_state(APPLIED_FILE, applied_entries)
|
||||
append_state(SKIPPED_FILE, skipped_entries)
|
||||
|
||||
if plan_data.get("upstream_head"):
|
||||
SEEN_FILE.write_text(plan_data["upstream_head"] + "\n", encoding="utf-8")
|
||||
|
||||
result = {
|
||||
"generated_at": utc_now(),
|
||||
"upstream_head": plan_data.get("upstream_head"),
|
||||
"applied": applied_entries,
|
||||
"skipped": skipped_entries,
|
||||
"ignored": ignored_entries,
|
||||
}
|
||||
write_json(Path(args.result), result)
|
||||
write_apply_report(Path(args.report), result)
|
||||
|
||||
print(
|
||||
f"Applied {len(applied_entries)} commits; "
|
||||
f"skipped {len(skipped_entries)} commits; "
|
||||
f"ignored {len(ignored_entries)} commits."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
def write_apply_report(path: Path, result: dict[str, Any]) -> None:
|
||||
lines = [
|
||||
"# Upstream Parser Sync Result",
|
||||
"",
|
||||
f"- Generated: {result['generated_at']}",
|
||||
f"- Upstream head: `{result.get('upstream_head') or 'unknown'}`",
|
||||
f"- Applied: {len(result['applied'])}",
|
||||
f"- Skipped: {len(result['skipped'])}",
|
||||
f"- Ignored: {len(result.get('ignored', []))}",
|
||||
"",
|
||||
"## Applied",
|
||||
"",
|
||||
]
|
||||
if not result["applied"]:
|
||||
lines.append("No commits were applied.")
|
||||
for item in result["applied"]:
|
||||
lines.append(f"- `{item['sha'][:12]}` {item['subject']}")
|
||||
|
||||
lines.extend(["", "## Skipped", ""])
|
||||
if not result["skipped"]:
|
||||
lines.append("No commits were skipped.")
|
||||
for item in result["skipped"]:
|
||||
lines.append(f"- `{item['sha'][:12]}` {item['subject']}: {item['reason']}")
|
||||
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
plan_parser = sub.add_parser("plan")
|
||||
plan_parser.add_argument("--upstream-ref", required=True)
|
||||
plan_parser.add_argument("--max-commits", type=int, default=20)
|
||||
plan_parser.add_argument("--output", default="upstream-sync-candidates.json")
|
||||
plan_parser.add_argument("--report", default="upstream-sync-plan.md")
|
||||
plan_parser.set_defaults(func=plan)
|
||||
|
||||
apply_parser = sub.add_parser("apply")
|
||||
apply_parser.add_argument("--plan", default="upstream-sync-candidates.json")
|
||||
apply_parser.add_argument("--decisions", default="upstream-sync-decisions.json")
|
||||
apply_parser.add_argument("--result", default="upstream-sync-result.json")
|
||||
apply_parser.add_argument("--report", default="upstream-sync-result.md")
|
||||
apply_parser.set_defaults(func=apply)
|
||||
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return args.func(args)
|
||||
except Exception as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user