GitVita
New repositoryImport repositoryNew issue
Your repositories Your dashboard Sign in

cloudflare/kumo

Public

mirrored fromhttps://github.com/cloudflare/kumoAvailable

Watch0Fork0Star0
CodeCommitsIssuesPull requestsActionsInsightsSecurity
d86c9318c15924a4d4fab2205271148f8f184454

Branches

Tags

  • No tags available.
0Branches0Tags
Go to file
Add file
Code

Clone

HTTPS

Download ZIP

kumo/ci/visual-regression

ci/visual-regression/run-visual-regression.ts

492lines · modecode

RawDownload
Latest commit unavailable.
1#!/usr/bin/env tsx
2import { execSync } from "node:child_process";
3import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
4import { join } from "node:path";
5import {
6 COMPONENT_ACTIONS,
7 discoverComponents,
8 getAffectedComponents,
9 type DiscoveredComponent,
10} from "./page-config";
11
12const WORKER_URL =
13 process.env.SCREENSHOT_WORKER_URL ??
14 "https://kumo-screenshot-worker.design-engineering.workers.dev";
15const SCREENSHOTS_DIR = "ci/visual-regression/screenshots";
16const API_KEY = process.env.SCREENSHOT_API_KEY ?? "";
17
18interface ScreenshotResult {
19 url: string;
20 image: string;
21 error?: string;
22 sectionId?: string;
23 sectionTitle?: string;
24}
25
26interface WorkerResponse {
27 results: ScreenshotResult[];
28}
29
30interface CapturedScreenshot {
31 id: string;
32 name: string;
33 path: string;
34 url: string | null;
35}
36
37interface ComparisonResult {
38 id: string;
39 name: string;
40 beforeUrl: string;
41 afterUrl: string;
42 changed: boolean;
43}
44
45function getChangedFiles(): string[] {
46 try {
47 const base = process.env.GITHUB_BASE_REF ?? "main";
48 const output = execSync(`git diff --name-only origin/${base}...HEAD`, {
49 encoding: "utf-8",
50 });
51 return output.trim().split("\n").filter(Boolean);
52 } catch {
53 return [];
54 }
55}
56
57function ensureDir(dir: string): void {
58 if (!existsSync(dir)) {
59 mkdirSync(dir, { recursive: true });
60 }
61}
62
63async function uploadImageToGitHub(
64 imageBuffer: Buffer,
65 filename: string,
66): Promise<string> {
67 const token = process.env.GITHUB_TOKEN;
68 const repo = process.env.GITHUB_REPOSITORY ?? "cloudflare/kumo";
69 const prNumber = process.env.GITHUB_PR_NUMBER ?? process.env.PR_NUMBER;
70 const runId = process.env.GITHUB_RUN_ID ?? Date.now().toString();
71
72 if (!token) {
73 throw new Error("GITHUB_TOKEN required for image upload");
74 }
75
76 const [owner, repoName] = repo.split("/");
77 const branch = `vr-screenshots-${prNumber}-${runId}`;
78 const path = `screenshots/${filename}`;
79
80 const mainRef = await fetch(
81 `https://api.github.com/repos/${owner}/${repoName}/git/ref/heads/main`,
82 {
83 headers: {
84 Authorization: `Bearer ${token}`,
85 Accept: "application/vnd.github.v3+json",
86 },
87 },
88 );
89 const mainData = (await mainRef.json()) as { object: { sha: string } };
90 const baseSha = mainData.object.sha;
91
92 const refCheck = await fetch(
93 `https://api.github.com/repos/${owner}/${repoName}/git/ref/heads/${branch}`,
94 {
95 headers: {
96 Authorization: `Bearer ${token}`,
97 Accept: "application/vnd.github.v3+json",
98 },
99 },
100 );
101
102 if (refCheck.status === 404) {
103 await fetch(`https://api.github.com/repos/${owner}/${repoName}/git/refs`, {
104 method: "POST",
105 headers: {
106 Authorization: `Bearer ${token}`,
107 Accept: "application/vnd.github.v3+json",
108 "Content-Type": "application/json",
109 },
110 body: JSON.stringify({
111 ref: `refs/heads/${branch}`,
112 sha: baseSha,
113 }),
114 });
115 }
116
117 const content = imageBuffer.toString("base64");
118
119 const existingFile = await fetch(
120 `https://api.github.com/repos/${owner}/${repoName}/contents/${path}?ref=${branch}`,
121 {
122 headers: {
123 Authorization: `Bearer ${token}`,
124 Accept: "application/vnd.github.v3+json",
125 },
126 },
127 );
128
129 const existingData = existingFile.ok
130 ? ((await existingFile.json()) as { sha?: string })
131 : null;
132
133 await fetch(
134 `https://api.github.com/repos/${owner}/${repoName}/contents/${path}`,
135 {
136 method: "PUT",
137 headers: {
138 Authorization: `Bearer ${token}`,
139 Accept: "application/vnd.github.v3+json",
140 "Content-Type": "application/json",
141 },
142 body: JSON.stringify({
143 message: `Visual regression: ${filename}`,
144 content,
145 branch,
146 ...(existingData?.sha ? { sha: existingData.sha } : {}),
147 }),
148 },
149 );
150
151 return `https://raw.githubusercontent.com/${owner}/${repoName}/${branch}/${path}`;
152}
153
154interface PageRequest {
155 url: string;
156 captureSections: boolean;
157 hideSidebar: boolean;
158 actions?: Array<{ type: string; selector: string; waitAfter?: number }>;
159}
160
161async function captureScreenshots(
162 baseUrl: string,
163 components: DiscoveredComponent[],
164 outputDir: string,
165 prefix: string,
166): Promise<CapturedScreenshot[]> {
167 ensureDir(outputDir);
168 const screenshots: CapturedScreenshot[] = [];
169
170 const requests: PageRequest[] = [];
171
172 for (const component of components) {
173 requests.push({
174 url: component.url,
175 captureSections: true,
176 hideSidebar: true,
177 });
178
179 const action = COMPONENT_ACTIONS[component.id];
180 if (action) {
181 requests.push({
182 url: component.url,
183 captureSections: false,
184 hideSidebar: true,
185 actions: [action],
186 });
187 }
188 }
189
190 console.log(`Capturing screenshots from ${baseUrl}...`);
191 console.log(` ${components.length} components, ${requests.length} requests`);
192
193 const headers: Record<string, string> = {
194 "Content-Type": "application/json",
195 };
196 if (API_KEY) {
197 headers["X-API-Key"] = API_KEY;
198 }
199
200 const response = await fetch(`${WORKER_URL}/batch`, {
201 method: "POST",
202 headers,
203 body: JSON.stringify({
204 baseUrl,
205 pages: requests,
206 viewport: { width: 1440, height: 900 },
207 hideSidebar: true,
208 }),
209 });
210
211 if (!response.ok) {
212 const text = await response.text();
213 throw new Error(`Worker request failed: ${response.status} - ${text}`);
214 }
215
216 const data = (await response.json()) as WorkerResponse;
217
218 for (const result of data.results) {
219 if (result.error) {
220 console.warn(` Error: ${result.url}: ${result.error}`);
221 continue;
222 }
223
224 if (!result.image) {
225 console.warn(` Empty: ${result.url}`);
226 continue;
227 }
228
229 const urlPath = new URL(result.url).pathname.replace(/\/$/, "");
230 const componentSlug = urlPath.split("/").pop() || "unknown";
231
232 const isOpenState = requests.some(
233 (r) =>
234 r.url === urlPath.replace(/\/$/, "") &&
235 r.actions &&
236 r.actions.length > 0,
237 );
238
239 let screenshotId: string;
240 let screenshotName: string;
241
242 if (result.sectionId) {
243 screenshotId = `${componentSlug}-${result.sectionId}`;
244 screenshotName = `${formatName(componentSlug)} / ${result.sectionTitle || result.sectionId}`;
245 } else if (isOpenState) {
246 screenshotId = `${componentSlug}-open`;
247 screenshotName = `${formatName(componentSlug)} (Open)`;
248 } else {
249 screenshotId = componentSlug;
250 screenshotName = formatName(componentSlug);
251 }
252
253 const filename = `${prefix}-${screenshotId}.png`;
254 const filepath = join(outputDir, filename);
255
256 const imageBuffer = Buffer.from(result.image, "base64");
257 writeFileSync(filepath, imageBuffer);
258
259 let imageUrl: string | null = null;
260 try {
261 imageUrl = await uploadImageToGitHub(imageBuffer, filename);
262 console.log(` OK: ${screenshotName} -> ${imageUrl}`);
263 } catch (err) {
264 const msg = err instanceof Error ? err.message : String(err);
265 if (msg.includes("GITHUB_TOKEN required")) {
266 console.log(` OK: ${screenshotName} (local only, no GITHUB_TOKEN)`);
267 } else {
268 console.error(` Upload failed for ${screenshotName}: ${msg}`);
269 }
270 }
271
272 screenshots.push({
273 id: screenshotId,
274 name: screenshotName,
275 path: filepath,
276 url: imageUrl,
277 });
278 }
279
280 return screenshots;
281}
282
283function formatName(slug: string): string {
284 return slug
285 .split("-")
286 .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
287 .join(" ");
288}
289
290function compareImages(beforePath: string, afterPath: string): boolean {
291 if (!existsSync(beforePath) || !existsSync(afterPath)) {
292 return true;
293 }
294
295 const before = readFileSync(beforePath);
296 const after = readFileSync(afterPath);
297
298 return !before.equals(after);
299}
300
301function generateMarkdownReport(comparisons: ComparisonResult[]): string {
302 const changed = comparisons.filter((c) => c.changed);
303 const unchanged = comparisons.filter((c) => !c.changed);
304
305 const lines: string[] = [
306 "<!-- kumo-visual-regression -->",
307 "## Visual Regression Report",
308 "",
309 ];
310
311 if (changed.length === 0) {
312 lines.push("No visual changes detected.");
313 return lines.join("\n");
314 }
315
316 lines.push(`**${changed.length} screenshot(s) with visual changes:**`);
317 lines.push("");
318
319 for (const comp of changed) {
320 lines.push(`### ${comp.name}`);
321 lines.push("");
322 lines.push("| Before | After |");
323 lines.push("|--------|-------|");
324 lines.push(`| ![Before](${comp.beforeUrl}) | ![After](${comp.afterUrl}) |`);
325 lines.push("");
326 }
327
328 if (unchanged.length > 0) {
329 lines.push("<details>");
330 lines.push(
331 `<summary>${unchanged.length} screenshot(s) unchanged</summary>`,
332 );
333 lines.push("");
334 unchanged.forEach((c) => lines.push(`- ${c.name}`));
335 lines.push("</details>");
336 }
337
338 lines.push("");
339 lines.push("---");
340 lines.push("*Generated by Kumo Visual Regression*");
341
342 return lines.join("\n");
343}
344
345async function postPRComment(body: string): Promise<void> {
346 const token = process.env.GITHUB_TOKEN;
347 const prNumber = process.env.GITHUB_PR_NUMBER ?? process.env.PR_NUMBER;
348 const repo = process.env.GITHUB_REPOSITORY ?? "cloudflare/kumo";
349
350 if (!token || !prNumber) {
351 console.log("Missing GITHUB_TOKEN or PR_NUMBER, skipping PR comment");
352 console.log("\n--- Report ---\n");
353 console.log(body);
354 return;
355 }
356
357 const [owner, repoName] = repo.split("/");
358 const marker = "<!-- kumo-visual-regression -->";
359
360 const commentsResponse = await fetch(
361 `https://api.github.com/repos/${owner}/${repoName}/issues/${prNumber}/comments`,
362 {
363 headers: {
364 Authorization: `Bearer ${token}`,
365 Accept: "application/vnd.github.v3+json",
366 },
367 },
368 );
369
370 const comments = (await commentsResponse.json()) as Array<{
371 id: number;
372 body?: string;
373 }>;
374 const existingComment = comments.find((c) => c.body?.startsWith(marker));
375
376 const url = existingComment
377 ? `https://api.github.com/repos/${owner}/${repoName}/issues/comments/${existingComment.id}`
378 : `https://api.github.com/repos/${owner}/${repoName}/issues/${prNumber}/comments`;
379
380 const method = existingComment ? "PATCH" : "POST";
381
382 await fetch(url, {
383 method,
384 headers: {
385 Authorization: `Bearer ${token}`,
386 Accept: "application/vnd.github.v3+json",
387 "Content-Type": "application/json",
388 },
389 body: JSON.stringify({ body }),
390 });
391
392 console.log(`PR comment ${existingComment ? "updated" : "created"}`);
393}
394
395async function main(): Promise<void> {
396 const args = process.argv.slice(2);
397 const fullRegression = args.includes("--full");
398
399 const beforeUrl = process.env.BEFORE_URL ?? "https://kumo-ui.com";
400 const afterUrl =
401 process.env.AFTER_URL ?? process.env.PREVIEW_URL ?? beforeUrl;
402
403 console.log("Discovering components from docs site...");
404 const allComponents = await discoverComponents(beforeUrl);
405 console.log(`Found ${allComponents.length} components\n`);
406
407 let components: DiscoveredComponent[];
408
409 if (fullRegression) {
410 components = allComponents;
411 console.log(
412 `Running full visual regression (${components.length} components)...\n`,
413 );
414 } else {
415 const changedFiles = getChangedFiles();
416 components = getAffectedComponents(changedFiles, allComponents);
417
418 if (components.length === 0) {
419 console.log(
420 "No relevant file changes detected. Skipping visual regression.",
421 );
422 return;
423 }
424
425 console.log(`Found ${components.length} affected component(s):`);
426 components.forEach((c) => console.log(` - ${c.name} (${c.url})`));
427 console.log("");
428 }
429
430 const beforeDir = join(SCREENSHOTS_DIR, "before");
431 const afterDir = join(SCREENSHOTS_DIR, "after");
432
433 console.log("=== Capturing BEFORE screenshots ===");
434 const beforeScreenshots = await captureScreenshots(
435 beforeUrl,
436 components,
437 beforeDir,
438 "before",
439 );
440
441 console.log("\n=== Capturing AFTER screenshots ===");
442 const afterScreenshots = await captureScreenshots(
443 afterUrl,
444 components,
445 afterDir,
446 "after",
447 );
448
449 console.log("\n=== Comparing screenshots ===");
450 const comparisons: ComparisonResult[] = [];
451
452 const beforeMap = new Map(beforeScreenshots.map((s) => [s.id, s]));
453 const afterMap = new Map(afterScreenshots.map((s) => [s.id, s]));
454
455 const allIds = Array.from(
456 new Set([...Array.from(beforeMap.keys()), ...Array.from(afterMap.keys())]),
457 );
458
459 for (const id of allIds) {
460 const before = beforeMap.get(id);
461 const after = afterMap.get(id);
462
463 if (!before || !after) continue;
464 if (!before.url || !after.url) {
465 console.log(
466 ` ${before?.name || after?.name || id}: skipped (upload failed)`,
467 );
468 continue;
469 }
470
471 const changed = compareImages(before.path, after.path);
472
473 comparisons.push({
474 id,
475 name: before.name,
476 beforeUrl: before.url,
477 afterUrl: after.url,
478 changed,
479 });
480
481 console.log(` ${before.name}: ${changed ? "CHANGED" : "unchanged"}`);
482 }
483
484 console.log("\n=== Generating report ===");
485 const report = generateMarkdownReport(comparisons);
486 await postPRComment(report);
487}
488
489main().catch((error) => {
490 console.error("Visual regression failed:", error);
491 process.exit(1);
492});
493