mirror of
https://github.com/github/awesome-copilot.git
synced 2026-08-07 18:03:02 +00:00
github-actions[bot]
532 lines
16 KiB
JavaScript
532 lines
16 KiB
JavaScript
import { createWriteStream } from "node:fs";
|
|
import {
|
|
mkdir,
|
|
readFile,
|
|
readdir,
|
|
rename,
|
|
rm,
|
|
stat,
|
|
statfs,
|
|
writeFile,
|
|
} from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
import { Readable, Transform } from "node:stream";
|
|
import { pipeline } from "node:stream/promises";
|
|
import {
|
|
ARTIFACT_DOWNLOAD_IDLE_TIMEOUT_MS,
|
|
CACHE_ROOT,
|
|
MAX_ARCHIVE_BYTES,
|
|
MAX_INLINE_PREVIEW_BYTES,
|
|
MIN_CACHE_FREE_BYTES,
|
|
} from "./constants.mjs";
|
|
import { CacheMaintenanceCoordinator } from "./cache-coordinator.mjs";
|
|
import { analyzeArtifact, hasRootIndexHtml } from "./detector.mjs";
|
|
import { getArtifact, openArtifactDownload } from "./github.mjs";
|
|
import { findZipEntry, readEntryPrefix, readZipEntry, readZipIndex } from "./zip.mjs";
|
|
|
|
const downloads = new Map();
|
|
const indexCache = new Map();
|
|
const maintenance = new CacheMaintenanceCoordinator();
|
|
let reservedDownloadBytes = 0;
|
|
let metadataWriteSequence = 0;
|
|
|
|
function normalizeArtifactId(value) {
|
|
const id = Number.parseInt(value, 10);
|
|
if (!Number.isSafeInteger(id) || id <= 0) {
|
|
throw new Error("Artifact id must be a positive integer.");
|
|
}
|
|
return String(id);
|
|
}
|
|
|
|
function pathsFor(value) {
|
|
const id = normalizeArtifactId(value);
|
|
return {
|
|
id,
|
|
archive: join(CACHE_ROOT, `${id}.zip`),
|
|
metadata: join(CACHE_ROOT, `${id}.json`),
|
|
};
|
|
}
|
|
|
|
async function writeJsonAtomic(path, value) {
|
|
const temporary = `${path}.${process.pid}.${++metadataWriteSequence}.tmp`;
|
|
try {
|
|
await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
|
await rename(temporary, path);
|
|
} catch (error) {
|
|
await rm(temporary, { force: true }).catch(() => {});
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function readMetadataFile(path) {
|
|
try {
|
|
return JSON.parse(await readFile(path, "utf8"));
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return null;
|
|
if (error instanceof SyntaxError) {
|
|
throw new Error(`Cached artifact metadata is invalid: ${path}`, { cause: error });
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function formatBytes(value) {
|
|
const bytes = Math.max(0, Number(value) || 0);
|
|
if (bytes < 1024) return `${bytes} B`;
|
|
const units = ["KiB", "MiB", "GiB", "TiB"];
|
|
let size = bytes;
|
|
let unit = -1;
|
|
do {
|
|
size /= 1024;
|
|
unit++;
|
|
} while (size >= 1024 && unit < units.length - 1);
|
|
return `${size.toFixed(size >= 10 ? 0 : 1)} ${units[unit]}`;
|
|
}
|
|
|
|
async function downloadCapacityBytes() {
|
|
const filesystem = await statfs(CACHE_ROOT);
|
|
const available = Number(filesystem.bavail) * Number(filesystem.bsize);
|
|
return Math.max(
|
|
0,
|
|
Math.min(
|
|
MAX_ARCHIVE_BYTES,
|
|
available - MIN_CACHE_FREE_BYTES - reservedDownloadBytes,
|
|
),
|
|
);
|
|
}
|
|
|
|
function assertDownloadFits(bytes, capacity) {
|
|
if (!Number.isFinite(bytes) || bytes <= 0) return;
|
|
if (bytes > MAX_ARCHIVE_BYTES) {
|
|
throw new Error("Artifact exceeds the 4 GiB ZIP download limit.");
|
|
}
|
|
if (bytes > capacity) {
|
|
throw new Error(
|
|
`Not enough free disk space to cache this artifact. ${formatBytes(bytes)} is required, with ${formatBytes(capacity)} available after the safety reserve.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
async function downloadArchive(
|
|
token,
|
|
repository,
|
|
artifact,
|
|
archivePath,
|
|
onProgress,
|
|
externalSignal = null,
|
|
) {
|
|
const declaredBytes = Number(artifact.sizeInBytes);
|
|
let reservedBytes = 0;
|
|
const reserve = async (totalBytes) => {
|
|
if (!Number.isFinite(totalBytes) || totalBytes <= reservedBytes) return;
|
|
assertDownloadFits(totalBytes, MAX_ARCHIVE_BYTES);
|
|
const additionalBytes = totalBytes - reservedBytes;
|
|
const capacity = await downloadCapacityBytes();
|
|
if (additionalBytes > capacity) {
|
|
throw new Error(
|
|
`Not enough free disk space to cache this artifact. ${formatBytes(totalBytes)} is required, with ${formatBytes(capacity + reservedBytes)} available after the safety reserve.`,
|
|
);
|
|
}
|
|
reservedDownloadBytes += additionalBytes;
|
|
reservedBytes = totalBytes;
|
|
};
|
|
const releaseReservation = () => {
|
|
reservedDownloadBytes = Math.max(0, reservedDownloadBytes - reservedBytes);
|
|
reservedBytes = 0;
|
|
};
|
|
await reserve(declaredBytes);
|
|
|
|
const controller = new AbortController();
|
|
if (externalSignal?.aborted) {
|
|
controller.abort(externalSignal.reason);
|
|
} else if (externalSignal) {
|
|
externalSignal.addEventListener("abort", () => controller.abort(externalSignal.reason), { once: true });
|
|
}
|
|
let idleTimer;
|
|
const resetIdleTimeout = () => {
|
|
clearTimeout(idleTimer);
|
|
idleTimer = setTimeout(() => {
|
|
controller.abort(new Error("Artifact download stalled for 60 seconds."));
|
|
}, ARTIFACT_DOWNLOAD_IDLE_TIMEOUT_MS);
|
|
};
|
|
resetIdleTimeout();
|
|
|
|
let response;
|
|
try {
|
|
response = await openArtifactDownload(token, repository, artifact.id, {
|
|
signal: controller.signal,
|
|
});
|
|
} catch (error) {
|
|
clearTimeout(idleTimer);
|
|
releaseReservation();
|
|
if (controller.signal.aborted && controller.signal.reason instanceof Error) {
|
|
throw controller.signal.reason;
|
|
}
|
|
throw error;
|
|
}
|
|
|
|
const expectedLength = Number.parseInt(response.headers.get("content-length") ?? "", 10);
|
|
try {
|
|
await reserve(expectedLength);
|
|
} catch (error) {
|
|
clearTimeout(idleTimer);
|
|
controller.abort(error);
|
|
releaseReservation();
|
|
throw error;
|
|
}
|
|
const totalBytes =
|
|
Number.isFinite(expectedLength) && expectedLength > 0
|
|
? expectedLength
|
|
: Number.isFinite(declaredBytes) && declaredBytes > 0
|
|
? declaredBytes
|
|
: null;
|
|
|
|
const temporary = `${archivePath}.${process.pid}.${Date.now()}.part`;
|
|
let received = 0;
|
|
let lastReportAt = 0;
|
|
let bytesPerSecond = 0;
|
|
const samples = [{ at: Date.now(), bytes: 0 }];
|
|
const report = (force = false) => {
|
|
if (!onProgress) return;
|
|
const now = Date.now();
|
|
if (!force && now - lastReportAt < 250) return;
|
|
samples.push({ at: now, bytes: received });
|
|
while (samples.length > 2 && samples[1].at < now - 5_000) samples.shift();
|
|
const first = samples[0];
|
|
const elapsedSeconds = (now - first.at) / 1_000;
|
|
bytesPerSecond =
|
|
elapsedSeconds >= 0.2 ? (received - first.bytes) / elapsedSeconds : 0;
|
|
const percent = totalBytes
|
|
? Math.min(100, (received / totalBytes) * 100)
|
|
: null;
|
|
onProgress({
|
|
artifactId: artifact.id,
|
|
artifactName: artifact.name,
|
|
stage: "downloading",
|
|
receivedBytes: received,
|
|
totalBytes,
|
|
percent,
|
|
bytesPerSecond,
|
|
etaSeconds:
|
|
totalBytes && bytesPerSecond > 0
|
|
? Math.max(0, (totalBytes - received) / bytesPerSecond)
|
|
: null,
|
|
});
|
|
lastReportAt = now;
|
|
};
|
|
report(true);
|
|
|
|
const limiter = new Transform({
|
|
transform(chunk, _encoding, callback) {
|
|
received += chunk.length;
|
|
if (received > MAX_ARCHIVE_BYTES) {
|
|
callback(new Error("Artifact exceeded the 4 GiB ZIP download limit."));
|
|
return;
|
|
}
|
|
if (reservedBytes > 0 && received > reservedBytes) {
|
|
callback(new Error("Artifact download exhausted the available cache space."));
|
|
return;
|
|
}
|
|
resetIdleTimeout();
|
|
report();
|
|
callback(null, chunk);
|
|
},
|
|
});
|
|
|
|
try {
|
|
await pipeline(
|
|
Readable.fromWeb(response.body),
|
|
limiter,
|
|
createWriteStream(temporary, { flags: "wx" }),
|
|
);
|
|
report(true);
|
|
await rename(temporary, archivePath);
|
|
} catch (error) {
|
|
await rm(temporary, { force: true });
|
|
if (controller.signal.aborted && controller.signal.reason instanceof Error) {
|
|
throw controller.signal.reason;
|
|
}
|
|
throw error;
|
|
} finally {
|
|
clearTimeout(idleTimer);
|
|
releaseReservation();
|
|
}
|
|
return {
|
|
bytesPerSecond,
|
|
receivedBytes: received,
|
|
totalBytes: totalBytes ?? received,
|
|
};
|
|
}
|
|
|
|
async function buildMetadata(token, repository, artifactId, onProgress, signal = null) {
|
|
const paths = pathsFor(artifactId);
|
|
const artifact = await getArtifact(token, repository, paths.id);
|
|
if (artifact.expired) {
|
|
throw new Error("This GitHub Actions artifact has expired.");
|
|
}
|
|
|
|
await mkdir(CACHE_ROOT, { recursive: true });
|
|
onProgress?.({
|
|
artifactId: artifact.id,
|
|
artifactName: artifact.name,
|
|
stage: "preparing",
|
|
receivedBytes: 0,
|
|
totalBytes: artifact.sizeInBytes,
|
|
percent: 0,
|
|
bytesPerSecond: 0,
|
|
etaSeconds: null,
|
|
});
|
|
const transfer = await downloadArchive(
|
|
token,
|
|
repository,
|
|
artifact,
|
|
paths.archive,
|
|
onProgress,
|
|
signal,
|
|
);
|
|
const compressedBytes = transfer.receivedBytes;
|
|
try {
|
|
onProgress?.({
|
|
artifactId: artifact.id,
|
|
artifactName: artifact.name,
|
|
stage: "indexing",
|
|
receivedBytes: compressedBytes,
|
|
totalBytes: transfer.totalBytes,
|
|
percent: 100,
|
|
bytesPerSecond: transfer.bytesPerSecond,
|
|
etaSeconds: 0,
|
|
});
|
|
const index = await readZipIndex(paths.archive);
|
|
const PREFIX_BYTES = 8 * 1024;
|
|
const analysis = await analyzeArtifact(index, async (entry) => {
|
|
return readEntryPrefix(paths.archive, entry, PREFIX_BYTES);
|
|
});
|
|
const timestamp = new Date().toISOString();
|
|
const metadata = {
|
|
artifact,
|
|
repository,
|
|
downloadedAt: timestamp,
|
|
lastAccessedAt: timestamp,
|
|
compressedBytes,
|
|
analysis,
|
|
};
|
|
await writeJsonAtomic(paths.metadata, metadata);
|
|
indexCache.set(paths.id, { archivePath: paths.archive, index });
|
|
onProgress?.({
|
|
artifactId: artifact.id,
|
|
artifactName: artifact.name,
|
|
stage: "ready",
|
|
receivedBytes: compressedBytes,
|
|
totalBytes: transfer.totalBytes,
|
|
percent: 100,
|
|
bytesPerSecond: transfer.bytesPerSecond,
|
|
etaSeconds: 0,
|
|
});
|
|
return metadata;
|
|
} catch (error) {
|
|
await Promise.all([
|
|
rm(paths.archive, { force: true }),
|
|
rm(paths.metadata, { force: true }),
|
|
]);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function cachedMetadata(artifactId) {
|
|
const paths = pathsFor(artifactId);
|
|
const metadata = await readMetadataFile(paths.metadata);
|
|
if (!metadata) return null;
|
|
try {
|
|
const archive = await stat(paths.archive);
|
|
if (!archive.isFile()) return null;
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") return null;
|
|
throw error;
|
|
}
|
|
return metadata;
|
|
}
|
|
|
|
export async function inspectArtifact(
|
|
token,
|
|
repository,
|
|
artifactId,
|
|
{ onProgress } = {},
|
|
) {
|
|
const paths = pathsFor(artifactId);
|
|
while (true) {
|
|
const barrier = maintenance.inspectionBarrier(paths.id);
|
|
if (!barrier) break;
|
|
await barrier;
|
|
}
|
|
const key = `${repository}:${paths.id}`;
|
|
const active = downloads.get(key);
|
|
if (active) {
|
|
if (onProgress) active.listeners.add(onProgress);
|
|
try {
|
|
return await active.operation;
|
|
} finally {
|
|
if (onProgress) active.listeners.delete(onProgress);
|
|
}
|
|
}
|
|
|
|
const listeners = new Set(onProgress ? [onProgress] : []);
|
|
const emit = (progress) => {
|
|
for (const listener of listeners) listener(progress);
|
|
};
|
|
const externalController = new AbortController();
|
|
const operation = (async () => {
|
|
const existing = await cachedMetadata(paths.id);
|
|
if (existing?.repository === repository && existing.analysis) {
|
|
existing.lastAccessedAt = new Date().toISOString();
|
|
await writeJsonAtomic(paths.metadata, existing);
|
|
emit({
|
|
artifactId: Number(paths.id),
|
|
artifactName: existing.artifact?.name ?? `Artifact ${paths.id}`,
|
|
stage: "ready",
|
|
receivedBytes: existing.compressedBytes,
|
|
totalBytes: existing.compressedBytes,
|
|
percent: 100,
|
|
bytesPerSecond: 0,
|
|
etaSeconds: 0,
|
|
cached: true,
|
|
});
|
|
return existing;
|
|
}
|
|
if (existing) {
|
|
indexCache.delete(paths.id);
|
|
await removeArtifactFiles(paths);
|
|
}
|
|
return buildMetadata(token, repository, paths.id, emit, externalController.signal);
|
|
})();
|
|
const download = { listeners, operation, abort: () => externalController.abort(new Error("Cache cleared.")) };
|
|
downloads.set(key, download);
|
|
try {
|
|
return await operation;
|
|
} catch (error) {
|
|
emit({
|
|
artifactId: Number(paths.id),
|
|
stage: "error",
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
throw error;
|
|
} finally {
|
|
if (downloads.get(key) === download) downloads.delete(key);
|
|
listeners.clear();
|
|
}
|
|
}
|
|
|
|
export async function getCachedArtifact(artifactId, { touch = true } = {}) {
|
|
const paths = pathsFor(artifactId);
|
|
const metadata = await cachedMetadata(paths.id);
|
|
if (!metadata) return null;
|
|
if (touch) {
|
|
metadata.lastAccessedAt = new Date().toISOString();
|
|
await writeJsonAtomic(paths.metadata, metadata);
|
|
}
|
|
return metadata;
|
|
}
|
|
|
|
async function getIndex(artifactId) {
|
|
const paths = pathsFor(artifactId);
|
|
const cached = indexCache.get(paths.id);
|
|
if (cached?.archivePath === paths.archive) return cached.index;
|
|
const index = await readZipIndex(paths.archive);
|
|
indexCache.set(paths.id, { archivePath: paths.archive, index });
|
|
return index;
|
|
}
|
|
|
|
export async function getCachedEntry(artifactId, entryPath) {
|
|
const paths = pathsFor(artifactId);
|
|
const metadata = await cachedMetadata(paths.id);
|
|
if (!metadata) throw new Error("Artifact is not cached. Inspect it first.");
|
|
const index = await getIndex(paths.id);
|
|
const entry = findZipEntry(index, entryPath);
|
|
if (!entry) throw new Error(`File was not found in the artifact: ${entryPath}`);
|
|
return { metadata, archivePath: paths.archive, index, entry };
|
|
}
|
|
|
|
export async function readCachedEntry(artifactId, entryPath, maxBytes = MAX_INLINE_PREVIEW_BYTES) {
|
|
const context = await getCachedEntry(artifactId, entryPath);
|
|
return {
|
|
...context,
|
|
content: await readZipEntry(context.archivePath, context.entry, maxBytes),
|
|
};
|
|
}
|
|
|
|
async function removeArtifactFiles(paths) {
|
|
await Promise.all([
|
|
rm(paths.archive, { force: true }),
|
|
rm(paths.metadata, { force: true }),
|
|
]);
|
|
}
|
|
|
|
export async function deleteCachedArtifact(artifactId) {
|
|
const paths = pathsFor(artifactId);
|
|
return maintenance.deleteArtifact(paths.id, async () => {
|
|
indexCache.delete(paths.id);
|
|
const suffix = `:${paths.id}`;
|
|
const matches = [...downloads.entries()]
|
|
.filter(([key]) => key.endsWith(suffix))
|
|
.map(([, entry]) => entry);
|
|
for (const entry of matches) entry.abort?.();
|
|
await Promise.allSettled(matches.map((entry) => entry.operation));
|
|
await removeArtifactFiles(paths);
|
|
return { deleted: paths.id };
|
|
});
|
|
}
|
|
|
|
export async function clearArtifactCache() {
|
|
return maintenance.clearCache(async () => {
|
|
const inflight = [...downloads.values()];
|
|
for (const entry of inflight) entry.abort?.();
|
|
await Promise.allSettled(inflight.map((entry) => entry.operation));
|
|
indexCache.clear();
|
|
await rm(CACHE_ROOT, { recursive: true, force: true });
|
|
await mkdir(CACHE_ROOT, { recursive: true });
|
|
return { cleared: true };
|
|
});
|
|
}
|
|
|
|
export async function getCacheSummary() {
|
|
await mkdir(CACHE_ROOT, { recursive: true });
|
|
const files = await readdir(CACHE_ROOT, { withFileTypes: true });
|
|
const metadataFiles = files.filter(
|
|
(entry) => entry.isFile() && /^\d+\.json$/.test(entry.name),
|
|
);
|
|
const artifacts = [];
|
|
const errors = [];
|
|
let totalBytes = 0;
|
|
|
|
for (const file of metadataFiles) {
|
|
const id = file.name.slice(0, -5);
|
|
try {
|
|
const metadata = await cachedMetadata(id);
|
|
if (!metadata) continue;
|
|
const archive = await stat(pathsFor(id).archive);
|
|
totalBytes += archive.size;
|
|
artifacts.push({
|
|
id,
|
|
name: metadata.artifact?.name ?? `Artifact ${id}`,
|
|
repository: metadata.repository,
|
|
bytes: archive.size,
|
|
downloadedAt: metadata.downloadedAt,
|
|
lastAccessedAt: metadata.lastAccessedAt,
|
|
primary:
|
|
["html", "static-site"].includes(metadata.analysis?.primary?.kind) &&
|
|
!hasRootIndexHtml(metadata.analysis?.entries ?? [])
|
|
? null
|
|
: metadata.analysis?.primary ?? null,
|
|
});
|
|
} catch (error) {
|
|
errors.push({
|
|
id,
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
|
|
artifacts.sort((left, right) =>
|
|
String(right.lastAccessedAt).localeCompare(String(left.lastAccessedAt)));
|
|
return { totalBytes, count: artifacts.length, artifacts, errors };
|
|
}
|