Merge pull request 'Optimize CMS images during builds' (#3) from review/cms-image-optimization into main

Reviewed-on: SloanOps/3031023-xyz#3

docs/decap-cms-gitea-setup.md

@@ -167,7 +167,9 @@ sites/www/content/**;sites/blog/content/**;public/images/uploads/*.jpg;public/im
 Do not add a nested upload glob. The current Decap configuration writes media
 directly into the flat uploads directory, and the local verifier rejects nested
 media paths. Gitea file patterns are case-insensitive; CI verifies actual file
-signatures and the 500 KiB limit before a preview can upload.
+signatures and a 25 MiB source safety ceiling before deployment. The build
+re-encodes the deployed copy to at most 1920 pixels and 500 KiB and strips
+embedded metadata.
 
 Everything not listed is protected from CMS/client pushes on `cms` and
 on every differently named branch, including:

docs/decap-cms-pilot.md

@@ -74,7 +74,8 @@ staging branch:
   replace, or remove an image reference while retaining accessible image
   descriptions and a layout fallback. URL insertion and SVG paths are
   rejected. Builds additionally verify the extension, raster signature, flat
-  upload path, and 500 KiB limit.
+  upload path, and a 25 MiB source safety ceiling. Every build re-encodes the
+  public copy to at most 1920 pixels and 500 KiB and removes embedded metadata.
 - `.gitea/workflows/cms-content-validation.yml` runs on every pull request into
   `main`, uses no secrets, and reports
   `CMS content validation / cms-content-validation (pull_request)` after
@@ -151,10 +152,16 @@ Do not expose `sites.json`, either `site.json`, schemas, workflows, Wrangler
 configuration, package files, source components, scripts, contact routing, or
 analytics through the CMS during this pilot.
 
-Configure the default media library with a conservative file-size limit. Do
+Configure image fields with a 25 MiB source safety limit so ordinary camera
-not accept executables, archives, HTML, scripts, or SVG uploads. Asset rights,
+photos can be uploaded. Do not accept executables, archives, HTML, scripts, or
-alt text, crop, dimensions, and optimization remain operator review items even
+SVG uploads. The repository verifier applies the same safety ceiling to every
-when the CMS accepts an upload.
+upload, including files added through Decap's standalone media gallery.
+
+The build re-encodes every uploaded raster without changing its public URL. It
+auto-orients and scales the deployed copy to at most 1920 pixels, targets a
+500 KiB output ceiling, and strips EXIF and other embedded metadata. The source
+file stays unchanged in the private repository. Asset rights, alt text, and
+crop remain operator review items.
 
 Decap's image widget applies its field `pattern` to the selected file's base
 name, not to the repository-relative media path. Keep that UI pattern limited
@@ -286,8 +293,8 @@ not record OAuth tokens or Cloudflare credentials.
 | DC-07 | Edit a primary FAQ | Only the primary preview rebuilds and displays the change. |
 | DC-08 | Edit service ordering and featured state | YAML types remain correct and the primary verifier passes. |
 | DC-09 | Add, replace, and remove a team portrait | The selected upload renders in the team card; removal uses the initials fallback. |
-| DC-10 | Upload a bounded raster image and select it in the fixed page-image record | File lands only in the approved media path and renders on the relevant primary preview page. |
+| DC-10 | Upload a camera-sized raster image and select it in the fixed page-image record | File lands only in the approved media path; its deployed copy is at most 1920 pixels and 500 KiB, contains no embedded metadata, and renders on the relevant primary preview page. |
-| DC-11 | Attempt an oversized upload and a disallowed file type | The CMS size limit rejects the oversized file before upload; Gitea's unprotected-extension patterns reject the disallowed path. |
+| DC-11 | Attempt an upload over 25 MiB and a disallowed file type | The source safety limit rejects the oversized file; Gitea's unprotected-extension patterns reject the disallowed path. |
 | DC-12 | Attempt to modify `.gitea/workflows/**`, source, package, script, schema, Wrangler, and site-manifest paths with the CMS account | Gitea rejects every push. |
 | DC-13 | Attempt to create a differently named branch and change a protected file | The fallback rule rejects the push; branch naming cannot bypass protection. |
 | DC-14 | Attempt a direct push to `main` | Gitea rejects it. |

package-lock.json

@@ -12,6 +12,7 @@
         "ajv-formats": "^3.0.1",
         "astro": "~7.2.0",
         "js-yaml": "4.3.2",
+        "sharp": "0.35.4",
         "wrangler": "^4.0.0"
       },
       "engines": {
@@ -4639,7 +4640,6 @@
       "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==",
       "dev": true,
       "license": "Apache-2.0",
-      "optional": true,
       "dependencies": {
         "@img/colour": "^1.1.0",
         "detect-libc": "^2.1.2",

package.json

@@ -19,6 +19,7 @@
     "ajv-formats": "^3.0.1",
     "astro": "~7.2.0",
     "js-yaml": "4.3.2",
+    "sharp": "0.35.4",
     "wrangler": "^4.0.0"
   }
 }

public/admin/config.yml

@@ -147,7 +147,7 @@ collections:
         pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
         media_library:
           config:
-            max_file_size: 512000
+            max_file_size: 26214400
             multiple: false
       - label: Display order
         name: order
@@ -183,7 +183,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: Home hero image description
             name: homeHeroImageAlt
             widget: string
@@ -196,7 +196,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: Home story image description
             name: homeStoryImageAlt
             widget: string
@@ -209,7 +209,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: About hero image description
             name: aboutHeroImageAlt
             widget: string
@@ -222,7 +222,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: About story image description
             name: aboutStoryImageAlt
             widget: string
@@ -235,7 +235,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: Services hero image description
             name: servicesHeroImageAlt
             widget: string
@@ -248,7 +248,7 @@ collections:
             allow_multiple: false
             pattern: ['^[A-Za-z0-9][A-Za-z0-9._-]*\.(?:jpe?g|png|webp|avif)$', 'Choose an approved JPG, PNG, WebP, or AVIF upload. SVG and general files are not allowed.']
             media_library:
-              config: { max_file_size: 512000, multiple: false }
+              config: { max_file_size: 26214400, multiple: false }
           - label: Contact hero image description
             name: contactHeroImageAlt
             widget: string

scripts/build-site.mjs

@@ -12,5 +12,8 @@ const result = spawnSync('npm', ['run', 'build', '--', '--outDir', `dist/${key}`
 });
 if (result.status !== 0) process.exit(result.status ?? 1);
 
+const images = spawnSync('node', ['scripts/optimize-images.mjs'], { env: { ...process.env, SITE_KEY: key }, stdio: 'inherit' });
+if (images.status !== 0) process.exit(images.status ?? 1);
+
 const seo = spawnSync('node', ['scripts/generate-seo.mjs'], { env: { ...process.env, SITE_KEY: key }, stdio: 'inherit' });
 if (seo.status !== 0) process.exit(seo.status ?? 1);

scripts/image-policy.mjs

@@ -0,0 +1,4 @@
+export const allowedUploadExtensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.avif']);
+export const maxSourceImageBytes = 25 * 1024 * 1024;
+export const maxBuiltImageBytes = 500 * 1024;
+export const maxBuiltImageDimension = 1920;

scripts/optimize-images.mjs

@@ -0,0 +1,75 @@
+import { readdir, readFile, writeFile } from 'node:fs/promises';
+import { extname, resolve } from 'node:path';
+import sharp from 'sharp';
+import {
+  allowedUploadExtensions,
+  maxBuiltImageBytes,
+  maxBuiltImageDimension,
+} from './image-policy.mjs';
+
+const key = process.env.SITE_KEY;
+if (!key || !/^[a-z0-9][a-z0-9-]{0,62}$/.test(key)) throw new Error('SITE_KEY is required to optimize built media.');
+
+const uploadRoot = resolve(`dist/${key}/images/uploads`);
+const entries = await readdir(uploadRoot, { withFileTypes: true });
+const qualitySteps = [82, 76, 70, 64, 58, 52, 46];
+const dimensionSteps = [1920, 1600, 1440, 1280, 1024, 800];
+
+function encode(pipeline, extension, quality) {
+  if (extension === '.jpg' || extension === '.jpeg') {
+    return pipeline.jpeg({ quality, progressive: true, chromaSubsampling: '4:2:0' });
+  }
+  if (extension === '.png') {
+    return pipeline.png({ compressionLevel: 9, effort: 7, palette: true, quality });
+  }
+  if (extension === '.webp') return pipeline.webp({ quality, effort: 6 });
+  if (extension === '.avif') return pipeline.avif({ quality, effort: 5, chromaSubsampling: '4:2:0' });
+  throw new Error(`Unsupported CMS image extension: ${extension}`);
+}
+
+async function optimize(source, extension) {
+  let smallest;
+  for (const dimension of dimensionSteps) {
+    for (const quality of qualitySteps) {
+      const pipeline = sharp(source, { failOn: 'warning', limitInputPixels: 100_000_000 })
+        .rotate()
+        .resize({
+          width: Math.min(dimension, maxBuiltImageDimension),
+          height: Math.min(dimension, maxBuiltImageDimension),
+          fit: 'inside',
+          withoutEnlargement: true,
+        });
+      const output = await encode(pipeline, extension, quality).toBuffer();
+      if (!smallest || output.length < smallest.length) smallest = output;
+      if (output.length <= maxBuiltImageBytes) return output;
+    }
+  }
+  throw new Error(`Unable to optimize an image below ${maxBuiltImageBytes} bytes; smallest result was ${smallest?.length ?? 0} bytes.`);
+}
+
+let inputBytes = 0;
+let outputBytes = 0;
+let optimizedCount = 0;
+for (const entry of entries) {
+  if (!entry.isFile()) throw new Error(`Nested or special CMS media paths are not allowed: ${entry.name}`);
+  const extension = extname(entry.name).toLowerCase();
+  if (!allowedUploadExtensions.has(extension)) throw new Error(`CMS media is not an allowed raster extension: ${entry.name}`);
+  const path = resolve(uploadRoot, entry.name);
+  const source = await readFile(path);
+  const output = await optimize(source, extension);
+  await writeFile(path, output);
+
+  const metadata = await sharp(output).metadata();
+  if ((metadata.width ?? 0) > maxBuiltImageDimension || (metadata.height ?? 0) > maxBuiltImageDimension) {
+    throw new Error(`Optimized image dimensions exceed ${maxBuiltImageDimension}px: ${entry.name}`);
+  }
+  if (metadata.exif || metadata.icc || metadata.iptc || metadata.xmp) {
+    throw new Error(`Optimized image retained metadata: ${entry.name}`);
+  }
+
+  inputBytes += source.length;
+  outputBytes += output.length;
+  optimizedCount += 1;
+}
+
+console.log(`Optimized ${optimizedCount} built CMS images from ${inputBytes} to ${outputBytes} bytes; metadata removed.`);

scripts/validate.mjs

@@ -2,6 +2,7 @@ import Ajv2020 from 'ajv/dist/2020.js';
 import addFormats from 'ajv-formats';
 import { readdir, readFile, stat } from 'node:fs/promises';
 import { extname, join, relative, sep } from 'node:path';
+import { allowedUploadExtensions, maxSourceImageBytes } from './image-policy.mjs';
 
 const readJson = async (path) => JSON.parse(await readFile(path, 'utf8'));
 const ajv = new Ajv2020({ allErrors: true, strict: true });
@@ -36,7 +37,6 @@ async function filesUnder(directory) {
 }
 
 const uploadRoot = join('public', 'images', 'uploads');
-const allowedUploadExtensions = new Set(['.jpg', '.jpeg', '.png', '.webp', '.avif']);
 const ascii = (value) => Buffer.from(value, 'ascii');
 const startsWith = (buffer, signature) => buffer.subarray(0, signature.length).equals(signature);
 const isRasterForExtension = (buffer, extension) => {
@@ -55,11 +55,10 @@ const isRasterForExtension = (buffer, extension) => {
 for (const path of await filesUnder('public/images')) {
   const imagePath = relative('public/images', path).split(sep).join('/');
   const info = await stat(path);
-  if (info.size > 500 * 1024) throw new Error(`Image exceeds the 500 KiB pilot limit: ${imagePath}`);
-
   const uploadPath = relative(uploadRoot, path);
   const isUpload = uploadPath !== '..' && !uploadPath.startsWith(`..${sep}`);
   if (!isUpload) continue;
+  if (info.size > maxSourceImageBytes) throw new Error(`CMS image exceeds the 25 MiB source safety limit: ${imagePath}`);
   if (uploadPath.includes(sep)) throw new Error(`Nested CMS media paths are not allowed: ${imagePath}`);
   const extension = extname(uploadPath).toLowerCase();
   if (!allowedUploadExtensions.has(extension)) throw new Error(`CMS media is not an allowed raster extension: ${imagePath}`);

scripts/verify-build.mjs

@@ -1,5 +1,7 @@
-import { access, readFile } from 'node:fs/promises';
+import { access, readFile, readdir, stat } from 'node:fs/promises';
-import { resolve } from 'node:path';
+import { extname, resolve } from 'node:path';
+import sharp from 'sharp';
+import { allowedUploadExtensions, maxBuiltImageBytes, maxBuiltImageDimension } from './image-policy.mjs';
 
 const key = process.env.SITE_KEY;
 if (!key) throw new Error('SITE_KEY is required.');
@@ -31,6 +33,19 @@ if (!admin.includes('decap-cms@3.15.1/dist/decap-cms.js')) throw new Error('Admi
 const headers = await readFile(resolve(root, '_headers'), 'utf8');
 if (!/\/admin\/\*[\s\S]*X-Robots-Tag: noindex, nofollow, noarchive/.test(headers)) throw new Error('Admin response headers are missing the no-index rule.');
 
+const builtUploads = resolve(root, 'images/uploads');
+for (const entry of await readdir(builtUploads, { withFileTypes: true })) {
+  if (!entry.isFile()) throw new Error(`Built CMS media contains a nested or special path: ${entry.name}.`);
+  const extension = extname(entry.name).toLowerCase();
+  if (!allowedUploadExtensions.has(extension)) throw new Error(`Built CMS media has a disallowed extension: ${entry.name}.`);
+  const path = resolve(builtUploads, entry.name);
+  const info = await stat(path);
+  if (info.size > maxBuiltImageBytes) throw new Error(`Built CMS image exceeds ${maxBuiltImageBytes} bytes: ${entry.name}.`);
+  const metadata = await sharp(path).metadata();
+  if ((metadata.width ?? 0) > maxBuiltImageDimension || (metadata.height ?? 0) > maxBuiltImageDimension) throw new Error(`Built CMS image exceeds ${maxBuiltImageDimension}px: ${entry.name}.`);
+  if (metadata.exif || metadata.icc || metadata.iptc || metadata.xmp) throw new Error(`Built CMS image retained metadata: ${entry.name}.`);
+}
+
 const home = await readFile(resolve(root, 'index.html'), 'utf8');
 if (key === 'www') {
   for (const { file, html } of pages) if (!html.includes('href="https://blog.3031023.xyz/"')) throw new Error(`${file} is missing the Happy Tails Journal link.`);

scripts/verify-decap.mjs

@@ -1,6 +1,7 @@
 import { readFile, readdir } from 'node:fs/promises';
 import { join } from 'node:path';
 import yaml from 'js-yaml';
+import { maxSourceImageBytes } from './image-policy.mjs';
 
 const fail = (message) => { throw new Error(message); };
 const assert = (condition, message) => { if (!condition) fail(message); };
@@ -54,7 +55,7 @@ const optionalImageField = {
   choose_url: false,
   allow_multiple: false,
   pattern: cmsImagePattern,
-  media_library: { config: { max_file_size: 512000, multiple: false } },
+  media_library: { config: { max_file_size: maxSourceImageBytes, multiple: false } },
 };
 const imageAltField = { widget: 'string', pattern: ['^.{8,180}$', 'Describe the image in 8–180 characters.'] };
 const expectedCollections = {
@@ -185,6 +186,13 @@ for (const fragment of [
   'featured: z.boolean().default(false)',
 ]) assert(schema.includes(fragment), `Astro schema changed; review the Decap contract for: ${fragment}`);
 
+const buildScript = await readFile('scripts/build-site.mjs', 'utf8');
+assert(buildScript.includes("['scripts/optimize-images.mjs']"), 'Every site build must optimize uploaded images before deployment.');
+const imageOptimizer = await readFile('scripts/optimize-images.mjs', 'utf8');
+for (const fragment of ['maxBuiltImageBytes', 'maxBuiltImageDimension', '.rotate()', 'metadata.exif', 'metadata.icc', 'metadata.iptc', 'metadata.xmp']) {
+  assert(imageOptimizer.includes(fragment), `Built-image optimization changed; review the CMS media contract for: ${fragment}`);
+}
+
 async function verifyContentPaths(directory) {
   const entries = await readdir(directory, { withFileTypes: true });
   for (const entry of entries) {