feat: surface external plugin review signals (#2623)

* feat: surface external plugin review signals

Add repository and homepage heuristics to external plugin intake and use eyes reactions for approval decisions.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: b745e915-6c5a-4354-ab77-5b52f9e66fea

* fix: harden external plugin review signals

Validate homepage destinations and bound response reads, and correct repository activity metrics.\n\nCo-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>\nCopilot-Session: b745e915-6c5a-4354-ab77-5b52f9e66fea

* fix: pin external homepage requests

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Copilot-Session: b745e915-6c5a-4354-ab77-5b52f9e66fea

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
Copilot-Session: b745e915-6c5a-4354-ab77-5b52f9e66fea
This commit is contained in:
Aaron Powell
2026-08-12 00:07:30 +00:00
committed by GitHub
parent dfde8ce6e7
commit 280b05dc88
3 changed files with 431 additions and 19 deletions
+317 -14
View File
@@ -2,7 +2,11 @@
import fs from "fs";
import path from "path";
import { lookup } from "node:dns/promises";
import http from "node:http";
import https from "node:https";
import { fileURLToPath } from "url";
import { isIP } from "node:net";
import { ROOT_FOLDER } from "./constants.mjs";
import { readExternalPlugins, validateExternalPlugin } from "./external-plugin-validation.mjs";
import { evaluateRefShaConsistency, normalizeCommitSha } from "./lib/external-plugin-source-ref-sha.mjs";
@@ -57,6 +61,15 @@ const LEGACY_FIELD_TITLES = Object.freeze({
});
const EXTERNAL_CANVAS_KEYWORD = "canvas";
const EXTERNAL_CANVAS_PREVIEW_PATH = "assets/preview.png";
const HOMEPAGE_FETCH_TIMEOUT_MS = 10_000;
const HOMEPAGE_MAX_BYTES = 512_000;
const HOMEPAGE_MAX_REDIRECTS = 5;
const MARKETING_SIGNAL_PATTERNS = Object.freeze([
["pricing", /\bpricing\b|\bplans?\b|\bsubscription\b|\bmonthly\b|\bannual\b/i],
["sales", /\bbook\s+a\s+demo\b|\bcontact\s+sales\b|\btalk\s+to\s+sales\b|\brequest\s+a\s+demo\b/i],
["trial", /\bfree\s+trial\b|\bstart\s+your\s+trial\b|\bget\s+started\s+free\b/i],
["checkout", /\bstripe\b|\bcheckout\b|\bsubscribe\s+now\b|\bbuy\s+now\b/i],
]);
const EXTERNAL_PLUGIN_ROOT_MANIFEST_PATHS = Object.freeze([
".github/plugin/plugin.json",
".plugin/plugin.json",
@@ -279,6 +292,251 @@ async function fetchGitHubFile(repo, filePath, ref, token) {
);
}
function isPublicAddress(address) {
if (isIP(address) === 4) {
const octets = address.split(".").map(Number);
const value = octets.reduce((result, octet) => (result * 256) + octet, 0);
return !(
octets[0] === 0 ||
octets[0] === 10 ||
octets[0] === 127 ||
(octets[0] === 100 && octets[1] >= 64 && octets[1] <= 127) ||
(octets[0] === 169 && octets[1] === 254) ||
(octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) ||
(octets[0] === 192 && octets[1] === 0 && octets[2] === 0) ||
(octets[0] === 192 && octets[1] === 0 && octets[2] === 2) ||
(octets[0] === 192 && octets[1] === 168) ||
(octets[0] === 198 && octets[1] >= 18 && octets[1] <= 19) ||
(octets[0] === 198 && octets[1] === 51 && octets[2] === 100) ||
(octets[0] === 203 && octets[1] === 0 && octets[2] === 113) ||
octets[0] >= 224 ||
value === 0xffffffff
);
}
if (isIP(address) !== 6) {
return false;
}
const normalized = address.toLowerCase().split("%")[0];
const embeddedIpv4 = normalized.slice(normalized.lastIndexOf(":") + 1);
if (isIP(embeddedIpv4) === 4 && !isPublicAddress(embeddedIpv4)) {
return false;
}
const groups = normalized.split("::");
const left = groups[0] ? groups[0].split(":") : [];
const right = groups[1] ? groups[1].split(":") : [];
const expanded = groups.length === 2
? [...left, ...Array(8 - left.length - right.length).fill("0"), ...right]
: left;
const values = expanded.map((group) => Number.parseInt(group || "0", 16));
const first = values[0] ?? 0;
const second = values[1] ?? 0;
const isMappedIpv4 = values.slice(0, 6).every((value, index) => value === (index === 5 ? 0xffff : 0));
return !(
values.every((value) => value === 0) ||
(values.slice(0, 7).every((value) => value === 0) && values[7] === 1) ||
(first & 0xfe00) === 0xfc00 ||
(first & 0xffc0) === 0xfe80 ||
(first & 0xff00) === 0xff00 ||
(first === 0x2001 && second === 0x0db8) ||
isMappedIpv4 && !isPublicAddress(
`${values[6] >> 8}.${values[6] & 0xff}.${values[7] >> 8}.${values[7] & 0xff}`,
)
);
}
async function assertPublicUrl(url) {
const addresses = await lookup(url.hostname, { all: true, verbatim: true });
if (!addresses.length || addresses.some(({ address }) => !isPublicAddress(address))) {
throw new Error("Homepage URL resolves to a non-public address.");
}
return addresses[0];
}
// Native fetch does not expose a lookup hook. This dispatcher connects to the
// address checked above while retaining the original hostname for Host/SNI.
export class PinnedAddressDispatcher {
constructor(url, address) {
this.url = url;
this.address = address;
}
dispatch(options, handler) {
const requestHeaders = Array.isArray(options.headers)
? Object.fromEntries(
Array.from({ length: options.headers.length / 2 }, (_, index) => [
options.headers[index * 2],
options.headers[index * 2 + 1],
]),
)
: { ...options.headers };
const defaultPort = this.url.protocol === "https:" ? "443" : "80";
if (!Object.keys(requestHeaders).some((name) => name.toLowerCase() === "host")) {
requestHeaders.Host = this.url.port && this.url.port !== defaultPort
? `${this.url.hostname}:${this.url.port}`
: this.url.hostname;
}
const requestOptions = {
protocol: this.url.protocol,
hostname: this.address.address,
port: this.url.port || defaultPort,
path: `${this.url.pathname}${this.url.search}`,
method: options.method,
headers: requestHeaders,
...(this.url.protocol === "https:" ? { servername: this.url.hostname } : {}),
};
const request = (this.url.protocol === "https:" ? https : http).request(requestOptions);
handler.onConnect?.(() => request.destroy());
request.once("response", (response) => {
const headers = response.rawHeaders;
if (handler.onHeaders(response.statusCode, headers, () => {}, response.statusMessage) === false) {
request.destroy();
return;
}
response.on("data", (chunk) => handler.onData(chunk));
response.once("end", () => handler.onComplete(null));
response.once("error", (error) => handler.onError(error));
});
request.once("error", (error) => handler.onError(error));
request.end();
return true;
}
close() {}
destroy() {}
}
async function readHomepageBody(response) {
if (!response.body?.getReader) {
throw new Error("Homepage response body was not readable.");
}
const reader = response.body.getReader();
const decoder = new TextDecoder();
const chunks = [];
let totalBytes = 0;
try {
while (totalBytes < HOMEPAGE_MAX_BYTES) {
const { done, value } = await reader.read();
if (done) break;
const remaining = HOMEPAGE_MAX_BYTES - totalBytes;
const chunk = value.byteLength > remaining ? value.subarray(0, remaining) : value;
chunks.push(decoder.decode(chunk, { stream: true }));
totalBytes += chunk.byteLength;
if (chunk.byteLength < value.byteLength) {
await reader.cancel();
break;
}
}
return chunks.join("") + decoder.decode();
} finally {
reader.releaseLock();
}
}
async function inspectHomepage(homepage) {
const result = {
status: "not_run",
url: homepage,
signals: [],
output: "",
};
if (!homepage) {
return result;
}
let parsedUrl;
try {
parsedUrl = new URL(homepage);
} catch {
return { ...result, status: "warning", output: "Homepage URL could not be parsed." };
}
if (!["http:", "https:"].includes(parsedUrl.protocol)) {
return { ...result, status: "warning", output: "Homepage URL uses an unsupported protocol." };
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), HOMEPAGE_FETCH_TIMEOUT_MS);
try {
let url = parsedUrl;
let response;
for (let redirectCount = 0; redirectCount <= HOMEPAGE_MAX_REDIRECTS; redirectCount += 1) {
const address = await assertPublicUrl(url);
response = await fetch(url, {
redirect: "manual",
headers: { Accept: "text/html,text/plain;q=0.9", "User-Agent": "awesome-copilot-external-plugin-intake" },
signal: controller.signal,
dispatcher: new PinnedAddressDispatcher(url, address),
});
if (response.status < 300 || response.status >= 400) break;
const location = response.headers?.get?.("location");
if (!location) break;
if (redirectCount === HOMEPAGE_MAX_REDIRECTS) {
return { ...result, status: "warning", output: "Homepage exceeded the redirect limit." };
}
url = new URL(location, url);
if (!["http:", "https:"].includes(url.protocol)) {
return { ...result, status: "warning", output: "Homepage URL uses an unsupported protocol." };
}
}
if (!response.ok) {
return { ...result, status: "warning", output: `Homepage returned HTTP ${response.status}.` };
}
const content = await readHomepageBody(response);
for (const [name, pattern] of MARKETING_SIGNAL_PATTERNS) {
if (pattern.test(content)) {
result.signals.push(name);
}
}
result.status = "pass";
result.output = result.signals.length
? `Detected homepage signals: ${result.signals.join(", ")}.`
: "No configured pricing or sales signals detected in the homepage content.";
return result;
} catch (error) {
return {
...result,
status: "warning",
output: error?.name === "AbortError" ? "Homepage inspection timed out." : `Homepage inspection failed: ${error.message}`,
};
} finally {
clearTimeout(timeout);
}
}
function buildRepositorySignals(repository) {
if (!repository) {
return { status: "not_run", output: "" };
}
const createdAt = repository.created_at ? new Date(repository.created_at) : null;
const ageDays = createdAt && !Number.isNaN(createdAt.valueOf())
? Math.max(0, Math.floor((Date.now() - createdAt.valueOf()) / 86_400_000))
: undefined;
const signals = [];
if (ageDays !== undefined && ageDays <= 14) signals.push(`repository is ${ageDays} day(s) old`);
if (repository.stargazers_count === 0) signals.push("0 stars");
if (repository.subscribers_count === 0) signals.push("0 watchers");
if (repository.forks_count === 0) signals.push("0 forks");
return {
status: "pass",
age_days: ageDays,
stars: repository.stargazers_count,
watchers: repository.subscribers_count,
forks: repository.forks_count,
open_issues: repository.open_issues_count,
signals,
output: signals.length ? `Detected repository signals: ${signals.join(", ")}.` : "No configured repository signals detected.",
};
}
function decodeGitHubFileContent(fileResponse) {
const encodedContent = fileResponse?.data?.content;
if (!encodedContent || typeof encodedContent !== "string") {
@@ -314,7 +572,7 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
if (repositoryResponse.kind === "notFound") {
errors.push(`submission: GitHub repository "${repo}" was not found`);
return;
return { status: "not_found", output: "" };
}
if (repositoryResponse.kind === "apiError") {
@@ -322,7 +580,7 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
warnings.push(
`submission: could not verify GitHub repository "${repo}" (${statusText}${repositoryResponse.reason ? `${repositoryResponse.reason}` : ""}); a maintainer should re-run intake`,
);
return;
return { status: "warning", output: `Repository metadata unavailable (${statusText}).` };
}
if (repositoryResponse.data?.private) {
@@ -362,7 +620,7 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
}
if (!ref) {
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (/^[0-9a-f]{40}$/i.test(ref)) {
@@ -377,15 +635,15 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
}
validateRefShaConsistency(normalizeCommitSha(ref));
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (ref.startsWith("refs/heads/") || ["main", "master", "develop", "development", "dev", "trunk"].includes(ref)) {
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (ref.startsWith("refs/") && !ref.startsWith("refs/tags/")) {
return;
return buildRepositorySignals(repositoryResponse.data);
}
const tagName = ref.startsWith("refs/tags/") ? ref.slice("refs/tags/".length) : ref;
@@ -393,13 +651,13 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
if (tagResponse.kind === "found") {
if (!normalizedSha) {
return;
return buildRepositorySignals(repositoryResponse.data);
}
const resolvedRefResponse = await resolveCommitSha(repo, ref, token);
if (resolvedRefResponse.kind === "notFound") {
errors.push(`submission: ref "${ref}" could not be resolved to a commit in GitHub repository "${repo}"`);
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (resolvedRefResponse.kind === "apiError") {
@@ -407,29 +665,29 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
errors.push(
`submission: ref "${ref}" does not resolve to a commit in GitHub repository "${repo}" (it may point to a tag object, tree, or blob); only commit-backed refs are supported`,
);
return;
return buildRepositorySignals(repositoryResponse.data);
}
const statusText = resolvedRefResponse.status ? `HTTP ${resolvedRefResponse.status}` : "network error";
warnings.push(
`submission: could not resolve ref "${ref}" to a commit in GitHub repository "${repo}" (${statusText}${resolvedRefResponse.reason ? `${resolvedRefResponse.reason}` : ""}); a maintainer should re-run intake`,
);
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (!resolvedRefResponse.commitSha) {
warnings.push(
`submission: could not determine the commit SHA for ref "${ref}" in GitHub repository "${repo}"; a maintainer should re-run intake`,
);
return;
return buildRepositorySignals(repositoryResponse.data);
}
validateRefShaConsistency(resolvedRefResponse.commitSha);
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (/^[0-9a-f]+$/i.test(ref) && ref.length !== 40) {
errors.push('submission: commit SHAs in "Ref to review" must use the full 40-character SHA or be submitted in "Commit SHA to review"');
return;
return buildRepositorySignals(repositoryResponse.data);
}
if (tagResponse.kind === "notFound") {
@@ -440,6 +698,7 @@ async function validateRemoteRepository(repo, { ref, sha }, errors, warnings, to
`submission: could not verify tag "${ref}" in GitHub repository "${repo}" (${statusText}${tagResponse.reason ? `${tagResponse.reason}` : ""}); a maintainer should re-run intake`,
);
}
return buildRepositorySignals(repositoryResponse.data);
}
function buildGitTreePath(repo, treeish, { recursive = false } = {}) {
@@ -815,6 +1074,7 @@ function buildQualityGatesCommentSection(qualityResult) {
if (status === "pass") {
return "✅ pass";
}
if (status === "warning" || (gate === "spec" && status === "fail")) {
return "⚠️ warning";
}
@@ -942,6 +1202,40 @@ function buildQualityGatesCommentSection(qualityResult) {
return sections.join("\n");
}
function buildReviewSignalsCommentSection(reviewSignals) {
const repository = reviewSignals?.repository;
const homepage = reviewSignals?.homepage;
if (!repository && !homepage) {
return "";
}
const rows = [
"### Reviewer signals",
"",
"_These are non-blocking heuristics for maintainer review; they are not evidence of misconduct or low quality._",
"",
"| Signal | Result |",
"|---|---|",
];
if (repository?.status === "pass") {
rows.push(
`| Repository age | ${repository.age_days === undefined ? "unknown" : `${repository.age_days} day(s)`} |`,
`| Repository activity | ${repository.signals?.length ? repository.signals.join("; ") : "no configured signal"} |`,
`| Repository counts | ${repository.stars ?? "unknown"} stars · ${repository.watchers ?? "unknown"} watchers · ${repository.forks ?? "unknown"} forks · ${repository.open_issues ?? "unknown"} open issues/PRs |`,
);
} else if (repository?.output) {
rows.push(`| Repository metadata | ${repository.output} |`);
}
if (homepage?.status === "pass") {
rows.push(`| Homepage heuristics | ${homepage.signals?.length ? `⚠️ ${homepage.signals.join(", ")}` : "no configured signal"} |`);
} else if (homepage?.output) {
rows.push(`| Homepage inspection | ${homepage.output} |`);
}
return rows.join("\n");
}
function getIntakeStateFromQualityResult(baseResult, qualityResult) {
if (!baseResult.valid) {
return "requires-submitter-fixes";
@@ -997,6 +1291,8 @@ function buildMergedIntakeComment(baseResult, qualityResult, runId, owner, repo)
baseResult.plugin?.source?.ref ? `- **Ref:** [\`${baseResult.plugin.source.ref.replaceAll('\`', '\\\`')}\`](https://github.com/${encodeRepoPath(baseResult.plugin.source.repo)}/tree/${encodeURIComponent(baseResult.plugin.source.ref).replaceAll("%2F", "/")})` : undefined,
baseResult.plugin?.source?.sha ? `- **SHA:** [\`${baseResult.plugin.source.sha.replaceAll('\`', '\\\`')}\`](https://github.com/${encodeRepoPath(baseResult.plugin.source.repo)}/tree/${encodeURIComponent(baseResult.plugin.source.sha).replaceAll("%2F", "/")})` : undefined,
"",
buildReviewSignalsCommentSection(baseResult.reviewSignals),
"",
qualitySection,
"",
"",
@@ -1033,6 +1329,7 @@ export async function evaluateExternalPluginIssue({ issue, token, runId, owner,
const parsed = parseExternalPluginIssueBody(issueBody);
const errors = [...parsed.errors];
const warnings = [];
let repositorySignals = { status: "not_run", output: "" };
const localPluginNames = readLocalPluginNames();
const { plugins: existingExternalPlugins } = readExternalPlugins({ policy: "marketplace" });
@@ -1056,8 +1353,10 @@ export async function evaluateExternalPluginIssue({ issue, token, runId, owner,
}
if (parsed.plugin?.source?.repo && (parsed.plugin?.source?.ref || parsed.plugin?.source?.sha)) {
await validateRemoteRepository(parsed.plugin.source.repo, parsed.plugin.source, errors, warnings, token);
repositorySignals = await validateRemoteRepository(parsed.plugin.source.repo, parsed.plugin.source, errors, warnings, token);
}
const homepageSignals = await inspectHomepage(parsed.plugin?.homepage);
const reviewSignals = { repository: repositorySignals, homepage: homepageSignals };
if (isCanvasPlugin) {
await validateCanvasPluginMetadata(parsed.plugin, errors, warnings, token);
@@ -1092,6 +1391,8 @@ export async function evaluateExternalPluginIssue({ issue, token, runId, owner,
parsed.plugin.source.sha ? `- **SHA:** [\`${parsed.plugin.source.sha.replaceAll('\`', '\\\`')}\`](https://github.com/${encodeRepoPath(parsed.plugin.source.repo)}/tree/${encodeURIComponent(parsed.plugin.source.sha).replaceAll("%2F", "/")})` : undefined,
`- **Keywords:** ${normalizedKeywords}`,
"",
buildReviewSignalsCommentSection(reviewSignals),
"",
"",
"### Canonical external.json payload",
"",
@@ -1117,6 +1418,7 @@ export async function evaluateExternalPluginIssue({ issue, token, runId, owner,
"### Required fixes",
"",
...dedupedErrors.map((error) => `- ${error}`),
buildReviewSignalsCommentSection(reviewSignals),
dedupedWarnings.length > 0
? ["", "### Warnings", "", ...dedupedWarnings.map((warning) => `- ${warning}`)].join("\n")
: "",
@@ -1130,6 +1432,7 @@ export async function evaluateExternalPluginIssue({ issue, token, runId, owner,
errors: dedupedErrors,
warnings: dedupedWarnings,
plugin: parsed.plugin,
reviewSignals,
isCanvasPlugin,
commentBody,
commentMarker: marker,