mirror of
https://github.com/github/awesome-copilot.git
synced 2026-06-15 04:15:03 +00:00
Add soft-gate PR risk scanning automation
Introduce a PR risk scanner script plus two workflows: one to scan changed files and upload findings, and one to upsert a sticky PR comment with a summary table and findings. This adds non-blocking supply-chain risk visibility for agentic contributions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Aaron Powell
@@ -0,0 +1,86 @@
|
||||
name: PR Risk Scan — Comment
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: ["PR Risk Scan — Gate"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
comment:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.workflow_run.event == 'pull_request'
|
||||
steps:
|
||||
- name: Download scan artifact
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: pr-risk-scan-results
|
||||
run-id: ${{ github.event.workflow_run.id }}
|
||||
github-token: ${{ github.token }}
|
||||
|
||||
- name: Upsert PR comment
|
||||
uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const marker = '<!-- pr-risk-scan-results -->';
|
||||
const reportPath = 'report.md';
|
||||
const prNumberPath = 'pr-number.txt';
|
||||
|
||||
if (!fs.existsSync(reportPath)) {
|
||||
core.setFailed('Risk scan report.md artifact was not found.');
|
||||
return;
|
||||
}
|
||||
|
||||
const body = fs.readFileSync(reportPath, 'utf8');
|
||||
let prNumber = null;
|
||||
|
||||
if (fs.existsSync(prNumberPath)) {
|
||||
const parsed = parseInt(fs.readFileSync(prNumberPath, 'utf8').trim(), 10);
|
||||
if (!Number.isNaN(parsed)) {
|
||||
prNumber = parsed;
|
||||
}
|
||||
}
|
||||
|
||||
if (!prNumber) {
|
||||
const fallback = context.payload.workflow_run.pull_requests?.[0]?.number;
|
||||
if (fallback) {
|
||||
prNumber = fallback;
|
||||
}
|
||||
}
|
||||
|
||||
if (!prNumber) {
|
||||
core.setFailed('Could not determine PR number for comment upsert.');
|
||||
return;
|
||||
}
|
||||
|
||||
const { data: comments } = await github.rest.issues.listComments({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
per_page: 100,
|
||||
});
|
||||
|
||||
const existing = comments.find((comment) => comment.body.includes(marker));
|
||||
|
||||
if (existing) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: existing.id,
|
||||
body,
|
||||
});
|
||||
console.log(`Updated existing risk scan comment ${existing.id}`);
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: prNumber,
|
||||
body,
|
||||
});
|
||||
console.log('Created new risk scan comment');
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
name: PR Risk Scan — Gate
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [staged]
|
||||
types: [opened, synchronize, reopened]
|
||||
paths:
|
||||
- "skills/**"
|
||||
- "agents/**"
|
||||
- "workflows/**"
|
||||
- "plugins/**"
|
||||
- "hooks/**"
|
||||
- "instructions/**"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Collect changed files
|
||||
run: |
|
||||
git diff --name-only --diff-filter=ACMR "origin/${{ github.base_ref }}...HEAD" > changed-files.txt
|
||||
echo "Changed files:"
|
||||
cat changed-files.txt || true
|
||||
|
||||
- name: Run PR risk scanner
|
||||
run: |
|
||||
mkdir -p pr-risk-results
|
||||
node ./eng/pr-risk-scan.mjs \
|
||||
--files changed-files.txt \
|
||||
--output-json pr-risk-results/results.json \
|
||||
--output-md pr-risk-results/report.md
|
||||
|
||||
- name: Save metadata
|
||||
run: |
|
||||
echo "${{ github.event.pull_request.number }}" > pr-risk-results/pr-number.txt
|
||||
|
||||
- name: Upload scan artifact
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
|
||||
with:
|
||||
name: pr-risk-scan-results
|
||||
path: pr-risk-results/
|
||||
retention-days: 1
|
||||
@@ -0,0 +1,311 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
const SCRIPT_EXTENSIONS = new Set([
|
||||
".sh",
|
||||
".bash",
|
||||
".ps1",
|
||||
".py",
|
||||
".js",
|
||||
".mjs",
|
||||
".ts",
|
||||
]);
|
||||
|
||||
const severityLevels = {
|
||||
high: "high",
|
||||
medium: "medium",
|
||||
info: "info",
|
||||
};
|
||||
|
||||
const LINE_RULES = [
|
||||
{
|
||||
rule_id: "guardrail-bypass-language",
|
||||
severity: severityLevels.high,
|
||||
regex:
|
||||
/\b(ignore (all|any|previous) (guardrails?|rules?|instructions?)|bypass (the )?(guardrails?|safety|policy)|disable (safety|guardrails?)|do not ask (for )?(confirmation|consent)|without prompting (the )?user)\b/i,
|
||||
reason: "Language suggests bypassing policy or confirmation controls.",
|
||||
suggested_fix:
|
||||
"Require explicit policy adherence and user-confirmation steps for risky actions.",
|
||||
},
|
||||
{
|
||||
rule_id: "remote-shell-execution",
|
||||
severity: severityLevels.high,
|
||||
regex: /\b(curl|wget)\b[^\n|]*\|\s*(sh|bash|zsh|pwsh|powershell)\b/i,
|
||||
reason: "Piping remote content directly to a shell is high-risk.",
|
||||
suggested_fix:
|
||||
"Download, verify integrity/signature, and run from a reviewed local file.",
|
||||
},
|
||||
{
|
||||
rule_id: "autoyes-package-exec",
|
||||
severity: severityLevels.high,
|
||||
regex:
|
||||
/\b(npx|npm\s+exec|pnpm\s+dlx|uvx|pipx\s+run)\b[^\n]*\s(-y|--yes)\b/i,
|
||||
reason:
|
||||
"Auto-yes execution can bypass human review of package/runtime prompts.",
|
||||
suggested_fix:
|
||||
"Remove automatic consent flags and require explicit reviewer-approved invocation.",
|
||||
},
|
||||
{
|
||||
rule_id: "package-exec-command",
|
||||
severity: severityLevels.medium,
|
||||
regex: /\b(npx|npm\s+exec|pnpm\s+dlx|uvx|pipx\s+run|uv\s+tool\s+run)\b/i,
|
||||
reason: "Dynamic package/runtime execution introduces supply-chain risk.",
|
||||
suggested_fix:
|
||||
"Pin exact versions and document manual confirmation controls.",
|
||||
},
|
||||
{
|
||||
rule_id: "unpinned-version-indicator",
|
||||
severity: severityLevels.medium,
|
||||
regex: /\B@latest\b|\blatest\b|\*|(\^|~)\d+/i,
|
||||
reason: "Unpinned dependencies can change behavior between runs.",
|
||||
suggested_fix: "Use exact immutable versions or commit hashes.",
|
||||
shouldApply: (line) =>
|
||||
/\b(npm|pnpm|yarn|npx|uvx|pip|pipx|cargo|go)\b/i.test(line),
|
||||
},
|
||||
];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const key = argv[i];
|
||||
if (!key.startsWith("--")) {
|
||||
continue;
|
||||
}
|
||||
|
||||
args[key.slice(2)] = argv[i + 1];
|
||||
i += 1;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function ensureParentDir(filePath) {
|
||||
const directory = path.dirname(filePath);
|
||||
fs.mkdirSync(directory, { recursive: true });
|
||||
}
|
||||
|
||||
function normalizeRelativePath(value) {
|
||||
const cleaned = String(value || "")
|
||||
.trim()
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/^\.\/+/, "");
|
||||
if (!cleaned) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (cleaned.includes("..")) {
|
||||
throw new Error(`Unsafe relative path in changed files list: ${value}`);
|
||||
}
|
||||
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
function isPotentialText(contentBuffer) {
|
||||
const nullByte = contentBuffer.includes(0x00);
|
||||
return !nullByte;
|
||||
}
|
||||
|
||||
function addFinding(findings, finding) {
|
||||
findings.push({
|
||||
rule_id: finding.rule_id,
|
||||
severity: finding.severity,
|
||||
file: finding.file,
|
||||
line: finding.line,
|
||||
match: finding.match.slice(0, 180),
|
||||
reason: finding.reason,
|
||||
suggested_fix: finding.suggested_fix,
|
||||
});
|
||||
}
|
||||
|
||||
function scanLineRules(filePath, content, findings) {
|
||||
const lines = content.split(/\r?\n/);
|
||||
for (let index = 0; index < lines.length; index += 1) {
|
||||
const line = lines[index];
|
||||
for (const rule of LINE_RULES) {
|
||||
if (typeof rule.shouldApply === "function" && !rule.shouldApply(line)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const match = line.match(rule.regex);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
|
||||
addFinding(findings, {
|
||||
rule_id: rule.rule_id,
|
||||
severity: rule.severity,
|
||||
file: filePath,
|
||||
line: index + 1,
|
||||
match: line.trim(),
|
||||
reason: rule.reason,
|
||||
suggested_fix: rule.suggested_fix,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function scanSkillScriptPath(filePath, findings) {
|
||||
const normalized = filePath.replace(/\\/g, "/");
|
||||
if (!normalized.startsWith("skills/")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const extension = path.extname(normalized).toLowerCase();
|
||||
if (!SCRIPT_EXTENSIONS.has(extension)) {
|
||||
return;
|
||||
}
|
||||
|
||||
addFinding(findings, {
|
||||
rule_id: "skill-script-added",
|
||||
severity: severityLevels.info,
|
||||
file: normalized,
|
||||
line: 1,
|
||||
match: normalized,
|
||||
reason:
|
||||
"Script asset under a skill may require external runtime/dependencies.",
|
||||
suggested_fix:
|
||||
"Document dependencies, pin versions, and avoid implicit network installs.",
|
||||
});
|
||||
}
|
||||
|
||||
function severityCounts(findings) {
|
||||
return findings.reduce(
|
||||
(acc, finding) => {
|
||||
acc[finding.severity] = (acc[finding.severity] || 0) + 1;
|
||||
return acc;
|
||||
},
|
||||
{ high: 0, medium: 0, info: 0 }
|
||||
);
|
||||
}
|
||||
|
||||
function toMarkdownReport(findings, scannedFiles, skippedFiles) {
|
||||
const marker = "<!-- pr-risk-scan-results -->";
|
||||
const counts = severityCounts(findings);
|
||||
const summary = [
|
||||
marker,
|
||||
"## 🔒 PR Risk Scan Results",
|
||||
"",
|
||||
`Scanned **${scannedFiles.length}** changed file(s).`,
|
||||
"",
|
||||
"| Severity | Count |",
|
||||
"|---|---:|",
|
||||
`| 🔴 High | ${counts.high} |`,
|
||||
`| 🟠 Medium | ${counts.medium} |`,
|
||||
`| ℹ️ Info | ${counts.info} |`,
|
||||
"",
|
||||
];
|
||||
|
||||
if (findings.length === 0) {
|
||||
summary.push(
|
||||
"✅ No matching risk patterns were detected in changed files."
|
||||
);
|
||||
} else {
|
||||
summary.push("| Severity | Rule | File | Line | Match |");
|
||||
summary.push("|---|---|---|---:|---|");
|
||||
for (const finding of findings.slice(0, 100)) {
|
||||
const severity =
|
||||
finding.severity === severityLevels.high
|
||||
? "🔴"
|
||||
: finding.severity === severityLevels.medium
|
||||
? "🟠"
|
||||
: "ℹ️";
|
||||
const match = finding.match.replace(/\|/g, "\\|");
|
||||
summary.push(
|
||||
`| ${severity} | \`${finding.rule_id}\` | \`${finding.file}\` | ${finding.line} | ${match} |`
|
||||
);
|
||||
}
|
||||
|
||||
if (findings.length > 100) {
|
||||
summary.push(
|
||||
"",
|
||||
`_${findings.length - 100} additional finding(s) omitted from table._`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (skippedFiles.length > 0) {
|
||||
summary.push(
|
||||
"",
|
||||
"<details>",
|
||||
"<summary>Skipped non-text or missing files</summary>",
|
||||
""
|
||||
);
|
||||
summary.push(skippedFiles.map((filePath) => `- ${filePath}`).join("\n"));
|
||||
summary.push("", "</details>");
|
||||
}
|
||||
|
||||
summary.push(
|
||||
"",
|
||||
"> This is an automated soft-gate report. Findings indicate review targets and do not block merge by themselves."
|
||||
);
|
||||
|
||||
return `${summary.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (!args.files || !args["output-json"] || !args["output-md"]) {
|
||||
throw new Error(
|
||||
"Usage: node ./eng/pr-risk-scan.mjs --files <changed-files.txt> --output-json <results.json> --output-md <report.md>"
|
||||
);
|
||||
}
|
||||
|
||||
const changedFilesPath = path.resolve(args.files);
|
||||
const outputJsonPath = path.resolve(args["output-json"]);
|
||||
const outputMarkdownPath = path.resolve(args["output-md"]);
|
||||
|
||||
const changedFiles = fs
|
||||
.readFileSync(changedFilesPath, "utf8")
|
||||
.split(/\r?\n/)
|
||||
.map(normalizeRelativePath)
|
||||
.filter(Boolean);
|
||||
|
||||
const findings = [];
|
||||
const scannedFiles = [];
|
||||
const skippedFiles = [];
|
||||
|
||||
for (const relativePath of changedFiles) {
|
||||
const absolutePath = path.resolve(relativePath);
|
||||
scanSkillScriptPath(relativePath, findings);
|
||||
|
||||
if (!fs.existsSync(absolutePath) || !fs.statSync(absolutePath).isFile()) {
|
||||
skippedFiles.push(relativePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
const contentBuffer = fs.readFileSync(absolutePath);
|
||||
if (!isPotentialText(contentBuffer)) {
|
||||
skippedFiles.push(relativePath);
|
||||
continue;
|
||||
}
|
||||
|
||||
const content = contentBuffer.toString("utf8");
|
||||
scanLineRules(relativePath, content, findings);
|
||||
scannedFiles.push(relativePath);
|
||||
}
|
||||
|
||||
const results = {
|
||||
generated_at: new Date().toISOString(),
|
||||
scanned_files: scannedFiles,
|
||||
skipped_files: skippedFiles,
|
||||
finding_count: findings.length,
|
||||
severity_counts: severityCounts(findings),
|
||||
findings,
|
||||
};
|
||||
|
||||
ensureParentDir(outputJsonPath);
|
||||
ensureParentDir(outputMarkdownPath);
|
||||
fs.writeFileSync(outputJsonPath, `${JSON.stringify(results, null, 2)}\n`);
|
||||
fs.writeFileSync(
|
||||
outputMarkdownPath,
|
||||
toMarkdownReport(findings, scannedFiles, skippedFiles)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
main();
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user