From dd6fae06c9abe17eb0f893c99ebcd3a5115942cd Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 4 Sep 2026 12:11:09 -0400 Subject: [PATCH] Optimize CMS images during builds --- docs/decap-cms-gitea-setup.md | 4 +- docs/decap-cms-pilot.md | 21 ++++++---- package-lock.json | 2 +- package.json | 1 + public/admin/config.yml | 14 +++---- scripts/build-site.mjs | 3 ++ scripts/image-policy.mjs | 4 ++ scripts/optimize-images.mjs | 75 +++++++++++++++++++++++++++++++++++ scripts/validate.mjs | 5 +-- scripts/verify-build.mjs | 19 ++++++++- scripts/verify-decap.mjs | 10 ++++- 11 files changed, 136 insertions(+), 22 deletions(-) create mode 100644 scripts/image-policy.mjs create mode 100644 scripts/optimize-images.mjs diff --git a/docs/decap-cms-gitea-setup.md b/docs/decap-cms-gitea-setup.md index 1726bf2..4e045cb 100644 --- a/docs/decap-cms-gitea-setup.md +++ b/docs/decap-cms-gitea-setup.md @@ -167,7 +167,9 @@ sites/www/content/**;sites/blog/content/**;public/images/uploads/*.jpg;public/im Do not add a nested upload glob. The current Decap configuration writes media directly into the flat uploads directory, and the local verifier rejects nested media paths. Gitea file patterns are case-insensitive; CI verifies actual file -signatures and the 500 KiB limit before a preview can upload. +signatures and a 25 MiB source safety ceiling before deployment. The build +re-encodes the deployed copy to at most 1920 pixels and 500 KiB and strips +embedded metadata. Everything not listed is protected from CMS/client pushes on `cms` and on every differently named branch, including: diff --git a/docs/decap-cms-pilot.md b/docs/decap-cms-pilot.md index dc2804e..a1ac224 100644 --- a/docs/decap-cms-pilot.md +++ b/docs/decap-cms-pilot.md @@ -74,7 +74,8 @@ staging branch: replace, or remove an image reference while retaining accessible image descriptions and a layout fallback. URL insertion and SVG paths are rejected. Builds additionally verify the extension, raster signature, flat - upload path, and 500 KiB limit. + upload path, and a 25 MiB source safety ceiling. Every build re-encodes the + public copy to at most 1920 pixels and 500 KiB and removes embedded metadata. - `.gitea/workflows/cms-content-validation.yml` runs on every pull request into `main`, uses no secrets, and reports `CMS content validation / cms-content-validation (pull_request)` after @@ -151,10 +152,16 @@ Do not expose `sites.json`, either `site.json`, schemas, workflows, Wrangler configuration, package files, source components, scripts, contact routing, or analytics through the CMS during this pilot. -Configure the default media library with a conservative file-size limit. Do -not accept executables, archives, HTML, scripts, or SVG uploads. Asset rights, -alt text, crop, dimensions, and optimization remain operator review items even -when the CMS accepts an upload. +Configure image fields with a 25 MiB source safety limit so ordinary camera +photos can be uploaded. Do not accept executables, archives, HTML, scripts, or +SVG uploads. The repository verifier applies the same safety ceiling to every +upload, including files added through Decap's standalone media gallery. + +The build re-encodes every uploaded raster without changing its public URL. It +auto-orients and scales the deployed copy to at most 1920 pixels, targets a +500 KiB output ceiling, and strips EXIF and other embedded metadata. The source +file stays unchanged in the private repository. Asset rights, alt text, and +crop remain operator review items. Decap's image widget applies its field `pattern` to the selected file's base name, not to the repository-relative media path. Keep that UI pattern limited @@ -286,8 +293,8 @@ not record OAuth tokens or Cloudflare credentials. | DC-07 | Edit a primary FAQ | Only the primary preview rebuilds and displays the change. | | DC-08 | Edit service ordering and featured state | YAML types remain correct and the primary verifier passes. | | DC-09 | Add, replace, and remove a team portrait | The selected upload renders in the team card; removal uses the initials fallback. | -| DC-10 | Upload a bounded raster image and select it in the fixed page-image record | File lands only in the approved media path and renders on the relevant primary preview page. | -| DC-11 | Attempt an oversized upload and a disallowed file type | The CMS size limit rejects the oversized file before upload; Gitea's unprotected-extension patterns reject the disallowed path. | +| DC-10 | Upload a camera-sized raster image and select it in the fixed page-image record | File lands only in the approved media path; its deployed copy is at most 1920 pixels and 500 KiB, contains no embedded metadata, and renders on the relevant primary preview page. | +| DC-11 | Attempt an upload over 25 MiB and a disallowed file type | The source safety limit rejects the oversized file; Gitea's unprotected-extension patterns reject the disallowed path. | | DC-12 | Attempt to modify `.gitea/workflows/**`, source, package, script, schema, Wrangler, and site-manifest paths with the CMS account | Gitea rejects every push. | | DC-13 | Attempt to create a differently named branch and change a protected file | The fallback rule rejects the push; branch naming cannot bypass protection. | | DC-14 | Attempt a direct push to `main` | Gitea rejects it. | diff --git a/package-lock.json b/package-lock.json index 02131bd..66c8ccb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "ajv-formats": "^3.0.1", "astro": "~7.2.0", "js-yaml": "4.3.2", + "sharp": "0.35.4", "wrangler": "^4.0.0" }, "engines": { @@ -4639,7 +4640,6 @@ "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", "dev": true, "license": "Apache-2.0", - "optional": true, "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", diff --git a/package.json b/package.json index 62aed0e..31e2cee 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "ajv-formats": "^3.0.1", "astro": "~7.2.0", "js-yaml": "4.3.2", + "sharp": "0.35.4", "wrangler": "^4.0.0" } } diff --git a/public/admin/config.yml b/public/admin/config.yml index 871ea60..e382633 100644 --- a/public/admin/config.yml +++ b/public/admin/config.yml @@ -147,7 +147,7 @@ collections: pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: config: - max_file_size: 512000 + max_file_size: 26214400 multiple: false - label: Display order name: order @@ -183,7 +183,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: Home hero image description name: homeHeroImageAlt widget: string @@ -196,7 +196,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: Home story image description name: homeStoryImageAlt widget: string @@ -209,7 +209,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: About hero image description name: aboutHeroImageAlt widget: string @@ -222,7 +222,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: About story image description name: aboutStoryImageAlt widget: string @@ -235,7 +235,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: Services hero image description name: servicesHeroImageAlt widget: string @@ -248,7 +248,7 @@ collections: allow_multiple: false pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.'] media_library: - config: { max_file_size: 512000, multiple: false } + config: { max_file_size: 26214400, multiple: false } - label: Contact hero image description name: contactHeroImageAlt widget: string diff --git a/scripts/build-site.mjs b/scripts/build-site.mjs index 13471c3..06fe8d8 100644 --- a/scripts/build-site.mjs +++ b/scripts/build-site.mjs @@ -12,5 +12,8 @@ const result = spawnSync('npm', ['run', 'build', '--', '--outDir', `dist/${key}` }); if (result.status !== 0) process.exit(result.status ?? 1); +const images = spawnSync('node', ['scripts/optimize-images.mjs'], { env: { ...process.env, SITE_KEY: key }, stdio: 'inherit' }); +if (images.status !== 0) process.exit(images.status ?? 1); + const seo = spawnSync('node', ['scripts/generate-seo.mjs'], { env: { ...process.env, SITE_KEY: key }, stdio: 'inherit' }); if (seo.status !== 0) process.exit(seo.status ?? 1); diff --git a/scripts/image-policy.mjs b/scripts/image-policy.mjs new file mode 100644 index 0000000..c9a5297 --- /dev/null +++ b/scripts/image-policy.mjs @@ -0,0 +1,4 @@ +export const allowedUploadExtensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.avif']); +export const maxSourceImageBytes = 25 * 1024 * 1024; +export const maxBuiltImageBytes = 500 * 1024; +export const maxBuiltImageDimension = 1920; diff --git a/scripts/optimize-images.mjs b/scripts/optimize-images.mjs new file mode 100644 index 0000000..061356c --- /dev/null +++ b/scripts/optimize-images.mjs @@ -0,0 +1,75 @@ +import { readdir, readFile, writeFile } from 'node:fs/promises'; +import { extname, resolve } from 'node:path'; +import sharp from 'sharp'; +import { + allowedUploadExtensions, + maxBuiltImageBytes, + maxBuiltImageDimension, +} from './image-policy.mjs'; + +const key = process.env.SITE_KEY; +if (!key || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(key)) throw new Error('SITE_KEY is required to optimize built media.'); + +const uploadRoot = resolve(`dist/${key}/images/uploads`); +const entries = await readdir(uploadRoot, { withFileTypes: true }); +const qualitySteps = [82, 76, 70, 64, 58, 52, 46]; +const dimensionSteps = [1920, 1600, 1440, 1280, 1024, 800]; + +function encode(pipeline, extension, quality) { + if (extension === '.jpg' || extension === '.jpeg') { + return pipeline.jpeg({ quality, progressive: true, chromaSubsampling: '4:2:0' }); + } + if (extension === '.png') { + return pipeline.png({ compressionLevel: 9, effort: 7, palette: true, quality }); + } + if (extension === '.webp') return pipeline.webp({ quality, effort: 6 }); + if (extension === '.avif') return pipeline.avif({ quality, effort: 5, chromaSubsampling: '4:2:0' }); + throw new Error(`Unsupported CMS image extension: ${extension}`); +} + +async function optimize(source, extension) { + let smallest; + for (const dimension of dimensionSteps) { + for (const quality of qualitySteps) { + const pipeline = sharp(source, { failOn: 'warning', limitInputPixels: 100_000_000 }) + .rotate() + .resize({ + width: Math.min(dimension, maxBuiltImageDimension), + height: Math.min(dimension, maxBuiltImageDimension), + fit: 'inside', + withoutEnlargement: true, + }); + const output = await encode(pipeline, extension, quality).toBuffer(); + if (!smallest || output.length < smallest.length) smallest = output; + if (output.length <= maxBuiltImageBytes) return output; + } + } + throw new Error(`Unable to optimize an image below ${maxBuiltImageBytes} bytes; smallest result was ${smallest?.length ?? 0} bytes.`); +} + +let inputBytes = 0; +let outputBytes = 0; +let optimizedCount = 0; +for (const entry of entries) { + if (!entry.isFile()) throw new Error(`Nested or special CMS media paths are not allowed: ${entry.name}`); + const extension = extname(entry.name).toLowerCase(); + if (!allowedUploadExtensions.has(extension)) throw new Error(`CMS media is not an allowed raster extension: ${entry.name}`); + const path = resolve(uploadRoot, entry.name); + const source = await readFile(path); + const output = await optimize(source, extension); + await writeFile(path, output); + + const metadata = await sharp(output).metadata(); + if ((metadata.width ?? 0) > maxBuiltImageDimension || (metadata.height ?? 0) > maxBuiltImageDimension) { + throw new Error(`Optimized image dimensions exceed ${maxBuiltImageDimension}px: ${entry.name}`); + } + if (metadata.exif || metadata.icc || metadata.iptc || metadata.xmp) { + throw new Error(`Optimized image retained metadata: ${entry.name}`); + } + + inputBytes += source.length; + outputBytes += output.length; + optimizedCount += 1; +} + +console.log(`Optimized ${optimizedCount} built CMS images from ${inputBytes} to ${outputBytes} bytes; metadata removed.`); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 8bcfe53..fd32807 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -2,6 +2,7 @@ import Ajv2020 from 'ajv/dist/2020.js'; import addFormats from 'ajv-formats'; import { readdir, readFile, stat } from 'node:fs/promises'; import { extname, join, relative, sep } from 'node:path'; +import { allowedUploadExtensions, maxSourceImageBytes } from './image-policy.mjs'; const readJson = async (path) => JSON.parse(await readFile(path, 'utf8')); const ajv = new Ajv2020({ allErrors: true, strict: true }); @@ -36,7 +37,6 @@ async function filesUnder(directory) { } const uploadRoot = join('public', 'images', 'uploads'); -const allowedUploadExtensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.avif']); const ascii = (value) => Buffer.from(value, 'ascii'); const startsWith = (buffer, signature) => buffer.subarray(0, signature.length).equals(signature); const isRasterForExtension = (buffer, extension) => { @@ -55,11 +55,10 @@ const isRasterForExtension = (buffer, extension) => { for (const path of await filesUnder('public/images')) { const imagePath = relative('public/images', path).split(sep).join('/'); const info = await stat(path); - if (info.size > 500 * 1024) throw new Error(`Image exceeds the 500 KiB pilot limit: ${imagePath}`); - const uploadPath = relative(uploadRoot, path); const isUpload = uploadPath !== '..' && !uploadPath.startsWith(`..${sep}`); if (!isUpload) continue; + if (info.size > maxSourceImageBytes) throw new Error(`CMS image exceeds the 25 MiB source safety limit: ${imagePath}`); if (uploadPath.includes(sep)) throw new Error(`Nested CMS media paths are not allowed: ${imagePath}`); const extension = extname(uploadPath).toLowerCase(); if (!allowedUploadExtensions.has(extension)) throw new Error(`CMS media is not an allowed raster extension: ${imagePath}`); diff --git a/scripts/verify-build.mjs b/scripts/verify-build.mjs index 9cef6ae..ff9e454 100644 --- a/scripts/verify-build.mjs +++ b/scripts/verify-build.mjs @@ -1,5 +1,7 @@ -import { access, readFile } from 'node:fs/promises'; -import { resolve } from 'node:path'; +import { access, readFile, readdir, stat } from 'node:fs/promises'; +import { extname, resolve } from 'node:path'; +import sharp from 'sharp'; +import { allowedUploadExtensions, maxBuiltImageBytes, maxBuiltImageDimension } from './image-policy.mjs'; const key = process.env.SITE_KEY; if (!key) throw new Error('SITE_KEY is required.'); @@ -31,6 +33,19 @@ if (!admin.includes('decap-cms@3.15.1/dist/decap-cms.js')) throw new Error('Admi const headers = await readFile(resolve(root, '_headers'), 'utf8'); if (!/\/admin\/\*[\s\S]*X-Robots-Tag: noindex, nofollow, noarchive/.test(headers)) throw new Error('Admin response headers are missing the no-index rule.'); +const builtUploads = resolve(root, 'images/uploads'); +for (const entry of await readdir(builtUploads, { withFileTypes: true })) { + if (!entry.isFile()) throw new Error(`Built CMS media contains a nested or special path: ${entry.name}.`); + const extension = extname(entry.name).toLowerCase(); + if (!allowedUploadExtensions.has(extension)) throw new Error(`Built CMS media has a disallowed extension: ${entry.name}.`); + const path = resolve(builtUploads, entry.name); + const info = await stat(path); + if (info.size > maxBuiltImageBytes) throw new Error(`Built CMS image exceeds ${maxBuiltImageBytes} bytes: ${entry.name}.`); + const metadata = await sharp(path).metadata(); + if ((metadata.width ?? 0) > maxBuiltImageDimension || (metadata.height ?? 0) > maxBuiltImageDimension) throw new Error(`Built CMS image exceeds ${maxBuiltImageDimension}px: ${entry.name}.`); + if (metadata.exif || metadata.icc || metadata.iptc || metadata.xmp) throw new Error(`Built CMS image retained metadata: ${entry.name}.`); +} + const home = await readFile(resolve(root, 'index.html'), 'utf8'); if (key === 'www') { for (const { file, html } of pages) if (!html.includes('href="https://blog.3031023.xyz/"')) throw new Error(`${file} is missing the Happy Tails Journal link.`); diff --git a/scripts/verify-decap.mjs b/scripts/verify-decap.mjs index 2b9a863..b7a8b86 100644 --- a/scripts/verify-decap.mjs +++ b/scripts/verify-decap.mjs @@ -1,6 +1,7 @@ import { readFile, readdir } from 'node:fs/promises'; import { join } from 'node:path'; import yaml from 'js-yaml'; +import { maxSourceImageBytes } from './image-policy.mjs'; const fail = (message) => { throw new Error(message); }; const assert = (condition, message) => { if (!condition) fail(message); }; @@ -54,7 +55,7 @@ const optionalImageField = { choose_url: false, allow_multiple: false, pattern: cmsImagePattern, - media_library: { config: { max_file_size: 512000, multiple: false } }, + media_library: { config: { max_file_size: maxSourceImageBytes, multiple: false } }, }; const imageAltField = { widget: 'string', pattern: ['^.{8,180}$', 'Describe the image in 8–180 characters.'] }; const expectedCollections = { @@ -185,6 +186,13 @@ for (const fragment of [ 'featured: z.boolean().default(false)', ]) assert(schema.includes(fragment), `Astro schema changed; review the Decap contract for: ${fragment}`); +const buildScript = await readFile('scripts/build-site.mjs', 'utf8'); +assert(buildScript.includes("['scripts/optimize-images.mjs']"), 'Every site build must optimize uploaded images before deployment.'); +const imageOptimizer = await readFile('scripts/optimize-images.mjs', 'utf8'); +for (const fragment of ['maxBuiltImageBytes', 'maxBuiltImageDimension', '.rotate()', 'metadata.exif', 'metadata.icc', 'metadata.iptc', 'metadata.xmp']) { + assert(imageOptimizer.includes(fragment), `Built-image optimization changed; review the CMS media contract for: ${fragment}`); +} + async function verifyContentPaths(directory) { const entries = await readdir(directory, { withFileTypes: true }); for (const entry of entries) {