Enforce external plugin ref/sha consistency (#2463)

* Enforce external plugin ref/sha consistency Extract shared ref/sha normalization and consistency checks into eng/lib and reuse them in intake plus quality gate flows. Add a dedicated ref/sha consistency quality gate surfaced in PR/intake summaries, and add targeted tests for matching and mismatched refs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6afe21ad-eafa-4c90-a1f2-053dedac7625 * Address review: tree/blob ref errors and PR workflow ref/sha column - resolveCommitShaAtReadRef: classify rev-parse failure as 'fail' instead of 'infra_error' because a successfully-fetched ref that doesn't dereference to a commit is a submitter problem, not infra. - validateRemoteRepository (intake): treat HTTP 422 from the commit endpoint as a submitter error; all other non-404 errors remain transient warnings requiring maintainer re-run. - external-plugin-pr-quality-gates.yml: add ref/sha consistency column to the per-plugin quality table and failure details block. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6afe21ad-eafa-4c90-a1f2-053dedac7625 --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6afe21ad-eafa-4c90-a1f2-053dedac7625
2026-08-01 23:12:29 +00:00 · 2026-07-29 15:28:13 +10:00
parent 1e14bd4faa
commit 8ae5a99109
7 changed files with 397 additions and 8 deletions
@@ -1,6 +1,6 @@
 import assert from "node:assert/strict";
-import { test } from "node:test";
-import { validateCanvasPluginMetadata } from "./external-plugin-intake.mjs";
+import { afterEach, test } from "node:test";
+import { evaluateExternalPluginIssue, validateCanvasPluginMetadata } from "./external-plugin-intake.mjs";

 const REPO = "owner/repo";
 const SHA = "0123456789abcdef0123456789abcdef01234567";
@@ -258,3 +258,146 @@ test("validateCanvasPluginMetadata accepts an entry point found within a truncat
  assert.deepEqual(errors, []);
  assert.deepEqual(warnings, []);
 });
+
+// ---------------------------------------------------------------------------
+// ref/sha consistency tests (evaluateExternalPluginIssue)
+// ---------------------------------------------------------------------------
+
+const ORIGINAL_FETCH = global.fetch;
+const INTAKE_REPO = "octo/example";
+const RESOLVED_REF_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
+const PROVIDED_SHA = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb";
+
+afterEach(() => {
+  global.fetch = ORIGINAL_FETCH;
+});
+
+function buildIssueBody({ ref, sha }) {
+  return [
+    "<!-- external-plugin-submission -->",
+    "### Plugin name",
+    "",
+    "intake-ref-sha-consistency-test-plugin",
+    "",
+    "### Short description",
+    "",
+    "Test plugin for external intake validation.",
+    "",
+    "### GitHub repository",
+    "",
+    INTAKE_REPO,
+    "",
+    "### Plugin path inside the repository",
+    "",
+    "_No response_",
+    "",
+    "### Ref to review",
+    "",
+    ref,
+    "",
+    "### Commit SHA to review",
+    "",
+    sha,
+    "",
+    "### Version",
+    "",
+    "1.2.3",
+    "",
+    "### License identifier",
+    "",
+    "MIT",
+    "",
+    "### Author name",
+    "",
+    "Copilot Test",
+    "",
+    "### Author URL",
+    "",
+    "_No response_",
+    "",
+    "### Homepage URL",
+    "",
+    "_No response_",
+    "",
+    "### Keywords",
+    "",
+    "testing",
+    "",
+    "### Additional notes for reviewers",
+    "",
+    "_No response_",
+    "",
+    "### Submission checklist",
+    "",
+    "- [x] The plugin lives in a public GitHub repository.",
+    "- [x] The ref and/or sha I provided is immutable (release tag and/or full 40-character commit SHA), not a branch.",
+    "- [x] This submission follows this repository's contribution, security, and responsible AI policies.",
+    "- [x] This plugin is not already listed in the Awesome Copilot marketplace.",
+    "",
+  ].join("\n");
+}
+
+function jsonResponse(payload, { status = 200 } = {}) {
+  return {
+    ok: status >= 200 && status < 300,
+    status,
+    statusText: status === 404 ? "Not Found" : "OK",
+    headers: new Map(),
+    async json() {
+      return payload;
+    },
+  };
+}
+
+function installMockFetch() {
+  global.fetch = async (url) => {
+    const requestUrl = String(url);
+    if (requestUrl === `https://api.github.com/repos/${INTAKE_REPO}`) {
+      return jsonResponse({ private: false, archived: false });
+    }
+
+    if (
+      requestUrl === `https://api.github.com/repos/${INTAKE_REPO}/git/commits/${PROVIDED_SHA}` ||
+      requestUrl === `https://api.github.com/repos/${INTAKE_REPO}/git/commits/${RESOLVED_REF_SHA}`
+    ) {
+      const sha = requestUrl.endsWith(RESOLVED_REF_SHA) ? RESOLVED_REF_SHA : PROVIDED_SHA;
+      return jsonResponse({ sha });
+    }
+
+    if (requestUrl === `https://api.github.com/repos/${INTAKE_REPO}/git/ref/tags/v1.2.3`) {
+      return jsonResponse({ object: { type: "tag", sha: "cccccccccccccccccccccccccccccccccccccccc" } });
+    }
+
+    if (requestUrl === `https://api.github.com/repos/${INTAKE_REPO}/commits/v1.2.3`) {
+      return jsonResponse({ sha: RESOLVED_REF_SHA });
+    }
+
+    return jsonResponse({}, { status: 404 });
+  };
+}
+
+test("evaluateExternalPluginIssue fails when ref and sha resolve to different commits", async () => {
+  installMockFetch();
+  const issue = { body: buildIssueBody({ ref: "v1.2.3", sha: PROVIDED_SHA }) };
+
+  const result = await evaluateExternalPluginIssue({ issue });
+
+  assert.equal(result.valid, false);
+  assert.match(
+    result.commentBody,
+    /must reference the same commit \(ref "v1\.2\.3" resolves to "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", sha is "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"\)/,
+  );
+});
+
+test("evaluateExternalPluginIssue passes when ref and sha resolve to the same commit", async () => {
+  installMockFetch();
+  const issue = { body: buildIssueBody({ ref: "v1.2.3", sha: RESOLVED_REF_SHA }) };
+
+  const result = await evaluateExternalPluginIssue({ issue });
+
+  assert.equal(result.valid, true);
+  assert.equal(
+    result.errors.some((error) => error.includes("must reference the same commit")),
+    false,
+  );
+});