A great model.
'); + assert.equal(result.provider, 'DeepSeek'); + assert.equal(result['context-window'], '128K'); + assert.equal(result['total-params'], '671B'); + assert.equal(result['active-params'], '37B'); + assert.equal(result.precision, 'FP8'); + assert.equal(result['model-url'], 'https://huggingface.co/deepseek-ai/DeepSeek-V3'); + assert.equal(result.live, true); + assert.equal(result.new, false); + assert.equal(result.trending, true); + assert.deepEqual(result.categories, ['cat-id-1', 'cat-id-2']); + assert.deepEqual(result.logo, logoField); + }); + + it('handles undefined optional fields', () => { + const model = { + display_name: 'Test', + name: 'test', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result.provider, ''); + assert.equal(result.description, ''); + assert.equal(result['model-url'], ''); + assert.equal(result.logo, null); + }); + + it('falls back to name when display_name is falsy', () => { + const model = { + name: 'test-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result.name, 'test-model'); + }); + + it('drops unknown modalities not in categoryMap', () => { + const model = { + display_name: 'Test', + name: 'test', + isLive: false, + isNew: false, + isTrending: false, + }; + const categoryMap = { llm: 'cat-1' }; + const modalities = ['LLM', 'UnknownModality']; + const result = toWebflowFields(model, modalities, categoryMap, null); + + assert.deepEqual(result.categories, ['cat-1']); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `node --test tests/fetch-models/transform.test.js` +Expected: FAIL — `toWebflowFields` not found + +- [ ] **Step 3: Write `toWebflowFields` in fetch-models.js** + +Add at the top of `scripts/fetch-models.js` (keep existing code below for now): + +```js +export function toWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + logo: logoField, + description: model.description ? `${model.description}
` : '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `node --test tests/fetch-models/transform.test.js` +Expected: PASS + +- [ ] **Step 5: Write failing tests for `diffModels`** + +```js +// tests/fetch-models/diff.test.js +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { diffModels } from '../../scripts/fetch-models.js'; + +describe('diffModels', () => { + it('identifies items to create', () => { + const apiModels = [{ slug: 'new-model', fields: { name: 'New' } }]; + const webflowItems = []; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 1); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 0); + }); + + it('identifies items to delete', () => { + const apiModels = []; + const webflowItems = [{ id: 'wf-1', fieldData: { slug: 'old-model', name: 'Old' } }]; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 1); + assert.equal(toDelete[0], 'wf-1'); + }); + + it('identifies items to update when fields differ', () => { + const apiModels = [{ slug: 'model-a', fields: { name: 'Model A', provider: 'New Co' } }]; + const webflowItems = [ + { id: 'wf-1', fieldData: { slug: 'model-a', name: 'Model A', provider: 'Old Co' } }, + ]; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 1); + assert.equal(toUpdate[0].id, 'wf-1'); + assert.equal(toDelete.length, 0); + }); + + it('skips unchanged items', () => { + const fields = { name: 'Model A', provider: 'Same Co', live: true }; + const apiModels = [{ slug: 'model-a', fields }]; + const webflowItems = [{ id: 'wf-1', fieldData: { slug: 'model-a', ...fields } }]; + const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 0); + assert.equal(unchanged, 1); + }); + + it('ignores logo field in comparison', () => { + const apiModels = [ + { slug: 'model-a', fields: { name: 'A', logo: { url: 'new.png', alt: 'A logo' } } }, + ]; + const webflowItems = [ + { + id: 'wf-1', + fieldData: { + slug: 'model-a', + name: 'A', + logo: { fileId: '123', url: 'https://cdn.webflow.com/old.png', alt: 'A logo' }, + }, + }, + ]; + const { toUpdate, unchanged } = diffModels(apiModels, webflowItems); + + assert.equal(toUpdate.length, 0); + assert.equal(unchanged, 1); + }); +}); +``` + +- [ ] **Step 6: Run tests to verify they fail** + +Run: `node --test tests/fetch-models/diff.test.js` +Expected: FAIL — `diffModels` not found + +- [ ] **Step 7: Write `diffModels`** + +Add to `scripts/fetch-models.js`: + +```js +const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (SKIP_DIFF_FIELDS.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} +``` + +- [ ] **Step 8: Run tests to verify they pass** + +Run: `node --test tests/fetch-models/diff.test.js` +Expected: PASS + +- [ ] **Step 9: Commit** + +```bash +git add tests/fetch-models/ scripts/fetch-models.js +git commit -m "feat: add toWebflowFields and diffModels with tests" +``` + +--- + +### Task 3: Rewrite fetch-models.js Main Script + +Replace the old orchestration with the new sync pipeline. Keep `fetchModelGarden` and `transformModel` (minus `pricing`), remove everything else. + +**Files:** +- Rewrite: `scripts/fetch-models.js` + +- [ ] **Step 1: Rewrite the full script** + +```js +// scripts/fetch-models.js +import { createHash } from 'node:crypto'; +import { createClient } from './webflow-api.js'; + +// -- Environment variables -- + +const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; +const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID } = process.env; + +if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { + console.error( + 'Missing required env vars: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL' + ); + process.exit(1); +} +if (!WEBFLOW_API_TOKEN || !WEBFLOW_SITE_ID) { + console.error('Missing required env vars: WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID'); + process.exit(1); +} + +const modularHeaders = { + 'X-Yatai-Api-Token': MODULAR_CLOUD_API_TOKEN, + 'X-Yatai-Organization': MODULAR_CLOUD_ORG, +}; + +const wf = createClient(WEBFLOW_API_TOKEN); + +// -- Modular Cloud API (unchanged) -- + +async function fetchModelGarden() { + const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { + headers: modularHeaders, + }); + if (!countRes.ok) throw new Error(`Count request failed: ${countRes.status}`); + const { total } = await countRes.json(); + + const listRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden?count=${total}`, { + headers: modularHeaders, + }); + if (!listRes.ok) throw new Error(`List request failed: ${listRes.status}`); + return listRes.json(); +} + +function transformModel(model) { + const meta = model.metadata || {}; + const tags = meta.tags || []; + return { + display_name: model.display_name, + name: model.name, + description: model.description, + model_id: model.model_id, + logo_url: meta.logo_url, + provider: meta.provider, + modalities: meta.modalities, + context_window: meta.context_window, + total_params: meta.total_params, + active_params: meta.active_params, + precision: meta.precision, + model_url: meta.model_url, + isLive: Boolean(model.gateway_id), + isNew: tags.includes('New'), + isTrending: tags.includes('Trending'), + }; +} + +// -- Field mapping -- + +export function toWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + logo: logoField, + description: model.description ? `${model.description}
` : '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} + +// -- Diff -- + +const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (SKIP_DIFF_FIELDS.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} + +// -- Logo resolution -- + +const MIME_TO_EXT = { + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/gif': '.gif', + 'image/webp': '.webp', +}; + +async function resolveLogo(model) { + const { logo_url, display_name, name } = model; + + if (!logo_url) return null; + + if (logo_url.startsWith('http')) { + return { url: logo_url, alt: `${display_name || name} logo` }; + } + + if (logo_url.startsWith('data:')) { + const match = logo_url.match(/^data:([^;]+);base64,(.+)$/s); + if (!match) return null; + + const [, mime, payload] = match; + const ext = MIME_TO_EXT[mime]; + if (!ext) return null; + + const buffer = Buffer.from(payload, 'base64'); + if (buffer.length === 0) return null; + + console.log(`Uploading logo for: ${name}`); + const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${name}${ext}`, buffer); + return { url: assetUrl, alt: `${display_name || name} logo` }; + } + + return null; +} + +// -- Categories sync -- + +async function syncCategories(models, categoriesCollectionId) { + const allModalities = new Set(); + for (const model of models) { + if (model.modalities) { + for (const m of model.modalities) allModalities.add(m); + } + } + + const existingItems = await wf.listCollectionItems(categoriesCollectionId); + const existingBySlug = new Map(); + for (const item of existingItems) { + existingBySlug.set(item.fieldData.slug, item.id); + } + + const categoryMap = {}; + for (const modality of allModalities) { + const slug = modality.toLowerCase(); + if (existingBySlug.has(slug)) { + categoryMap[slug] = existingBySlug.get(slug); + } else { + console.log(`Creating category: ${modality}`); + const result = await wf.createItems(categoriesCollectionId, [ + { name: modality, slug }, + ]); + const newItem = result.items[0]; + categoryMap[slug] = newItem.id; + await wf.publishItems(categoriesCollectionId, [newItem.id]); + } + } + + return categoryMap; +} + +// -- Main -- + +async function main() { + // Phase 1: Fetch + console.log('Fetching models from Modular Cloud API...'); + const modelGarden = await fetchModelGarden(); + const models = modelGarden.items.map(transformModel); + console.log(`Fetched ${models.length} models`); + + // Discover collections + const collections = await wf.getCollections(WEBFLOW_SITE_ID); + const categoriesCol = wf.findCollectionBySlug(collections, 'models-categories'); + const modelsCol = wf.findCollectionBySlug(collections, 'models'); + + // Sync categories + console.log('Syncing categories...'); + const categoryMap = await syncCategories(models, categoriesCol.id); + console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); + + // Resolve logos and build field data + console.log('Resolving logos and building field data...'); + const apiModels = []; + for (const model of models) { + const logoField = await resolveLogo(model); + const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); + apiModels.push({ slug: model.name, fields }); + } + + // Phase 2: Diff + console.log('Fetching existing Webflow items...'); + const webflowItems = await wf.listCollectionItems(modelsCol.id); + const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); + + // Phase 3: Sync + const publishIds = []; + + if (toCreate.length > 0) { + console.log(`Creating ${toCreate.length} models...`); + for (const fields of toCreate) { + console.log(` Creating: ${fields.slug}`); + } + const created = await wf.createItems(modelsCol.id, toCreate); + publishIds.push(...created.items.map((item) => item.id)); + } + + if (toUpdate.length > 0) { + console.log(`Updating ${toUpdate.length} models...`); + for (const item of toUpdate) { + console.log(` Updating: ${item.fieldData.slug}`); + } + await wf.updateItems(modelsCol.id, toUpdate); + publishIds.push(...toUpdate.map((item) => item.id)); + } + + if (toDelete.length > 0) { + console.log(`Deleting ${toDelete.length} models...`); + await wf.deleteItems(modelsCol.id, toDelete); + } + + if (publishIds.length > 0) { + console.log(`Publishing ${publishIds.length} items...`); + await wf.publishItems(modelsCol.id, publishIds); + } + + // Summary + console.log( + `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` + ); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); +``` + +- [ ] **Step 2: Verify tests still pass** + +Run: `node --test tests/fetch-models/transform.test.js tests/fetch-models/diff.test.js` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add scripts/fetch-models.js +git commit -m "feat: rewrite fetch-models.js for Webflow CMS sync" +``` + +--- + +### Task 4: Update GitHub Actions Workflow + +**Files:** +- Modify: `.github/workflows/fetch-models.yml` + +- [ ] **Step 1: Rewrite the workflow** + +```yaml +name: Fetch Model Garden + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + environment: + description: 'Target environment' + type: choice + options: + - test + - production + default: 'test' + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Sync models to Webflow + run: node scripts/fetch-models.js + env: + MODULAR_CLOUD_API_TOKEN: ${{ secrets.MODULAR_CLOUD_API_TOKEN }} + MODULAR_CLOUD_ORG: ${{ vars.MODULAR_CLOUD_ORG }} + MODULAR_CLOUD_BASE_URL: ${{ vars.MODULAR_CLOUD_BASE_URL }} + WEBFLOW_API_TOKEN: ${{ inputs.environment == 'production' && secrets.WEBFLOW_API_TOKEN || secrets.TEST_WEBFLOW_API_TOKEN }} + WEBFLOW_SITE_ID: ${{ inputs.environment == 'production' && vars.WEBFLOW_SITE_ID || vars.TEST_WEBFLOW_SITE_ID }} +``` + +Key changes: +- Removed `permissions: contents: write` (no longer writing to repo) +- Removed the `git commit` / `git push` step entirely +- Added `workflow_dispatch` inputs with environment choice +- Added conditional env var selection for test vs production +- Renamed job from `fetch` to `sync` +- **Scheduled runs** (cron): `inputs.environment` is undefined, so the conditional falls through to test secrets — scheduled runs always target the test site + +- [ ] **Step 2: Commit** + +```bash +git add .github/workflows/fetch-models.yml +git commit -m "feat: update workflow for Webflow CMS sync with environment selection" +``` + +--- + +### Task 5: Clean Up Old Files + +Remove files and directories that are no longer needed. + +**Files:** +- Delete: `data/models.json` +- Delete: `data/images/` (entire directory) + +- [ ] **Step 1: Remove old data files** + +```bash +rm data/models.json +rm -rf data/images/ +``` + +- [ ] **Step 2: Verify nothing references the removed files** + +Search the codebase for references to `models.json` or `data/images`: + +```bash +grep -r "models.json" --include="*.js" --include="*.ts" --include="*.yml" . +grep -r "data/images" --include="*.js" --include="*.ts" --include="*.yml" . +``` + +Expected: No results (or only references in the spec/plan docs). + +- [ ] **Step 3: Commit** + +```bash +git add -A data/ +git commit -m "chore: remove models.json and data/images (replaced by Webflow CMS)" +``` + +--- + +### Task 6: Integration Test Against Test Site + +Run the script against the real Webflow test site to verify end-to-end functionality. + +**Files:** +- No new files; uses existing script + +- [ ] **Step 1: Run the schema migration (Task 0) if not done yet** + +Ensure the `logo` field on the test site Models collection is type Image, not Link. + +- [ ] **Step 2: Set environment variables locally** + +```bash +export MODULAR_CLOUD_API_TOKEN="..." +export MODULAR_CLOUD_ORG="..." +export MODULAR_CLOUD_BASE_URL="..." +export WEBFLOW_API_TOKEN="..." # Test site token +export WEBFLOW_SITE_ID="696947140cab938ac6990602" # Test site only! +``` + +- [ ] **Step 3: Run the script** + +```bash +node scripts/fetch-models.js +``` + +Expected output: +``` +Fetching models from Modular Cloud API... +Fetched N models +Syncing categories... +Categories ready: llm, vision, audio, image +Resolving logos and building field data... +Fetching existing Webflow items... +Creating X models... +Updating Y models... +Deleting Z models... +Publishing N items... + +Sync complete. Created: X, Updated: Y, Deleted: Z, Unchanged: W +``` + +- [ ] **Step 4: Verify in Webflow** + +Check the test site CMS in the Webflow dashboard: +- Categories collection has the expected items +- Models collection has the expected items with all fields populated +- Logo images are displaying correctly +- MultiReference links between models and categories are correct + +- [ ] **Step 5: Run the script again (idempotency check)** + +```bash +node scripts/fetch-models.js +``` + +Expected: `Created: 0, Updated: 0, Deleted: 0, Unchanged: N` — second run should be a no-op. + +- [ ] **Step 6: Commit any final fixes** + +If any fixes were needed during integration testing, commit them: + +```bash +git add scripts/ +git commit -m "fix: address issues found during integration testing" +``` diff --git a/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md new file mode 100644 index 0000000..df968be --- /dev/null +++ b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md @@ -0,0 +1,228 @@ +# Webflow CMS Sync Design + +**Date:** 2026-03-20 +**Status:** Draft +**Scope:** Replace `models.json` file output with direct Webflow CMS sync via Data API + +## Summary + +Modify `scripts/fetch-models.js` to sync model data from the Modular Cloud Model Garden API directly into Webflow CMS collections, replacing the current `models.json` + `data/images/` + jsDelivr pipeline. The Webflow CMS becomes the sole output of the script. + +**Target site:** Test Site - Blog Integration (`696947140cab938ac6990602`). The production Modular site must never be touched by this script. + +## Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Output target | Webflow CMS only, no `models.json` | CMS is the source of truth | +| Auth | Webflow Data API token (env var) | No MCP dependency in CI | +| Category management | Hybrid -- reuse existing, auto-create new, never delete | New modalities appear organically | +| Model deletion | Delete from Webflow if absent from API | CMS mirrors API exactly | +| Matching key | `slug` (derived from `model.name`) | Stable, unique, Webflow-native lookup | +| Sync strategy | Single-pass batch diff | Minimizes API calls (~5 per run for ~35 models) | +| Image handling | Upload base64 to Webflow Assets API; pass URLs directly to Image field | Eliminates jsDelivr/git image pipeline | +| Error handling | Fail hard (exit 1) on any error, log summary | Clear signal in GitHub Actions | +| Site configurability | Workflow `environment` input selects test vs production config | Supports future production cutover | + +## Architecture + +### Three-Phase Pipeline + +``` +Modular Cloud API --fetch--> Transform --diff--> Webflow CMS + | + Resolve logos + (URL passthrough + or Asset API upload) +``` + +**Phase 1 -- Fetch:** Pull all models from Modular Cloud API (existing logic, unchanged). + +**Phase 2 -- Diff:** Fetch all existing Webflow CMS items (categories + models), compare against API data, produce three lists: to-create, to-update, to-delete. + +**Phase 3 -- Sync:** Execute batch create/update/delete against Webflow, then publish created and updated items. + +### Environment Variables + +| Variable | Type | Purpose | +|---|---|---| +| `MODULAR_CLOUD_API_TOKEN` | Secret | Existing -- Model Garden API auth | +| `MODULAR_CLOUD_ORG` | Var | Existing -- Model Garden org | +| `MODULAR_CLOUD_BASE_URL` | Var | Existing -- Model Garden endpoint | +| `WEBFLOW_API_TOKEN` | Secret | New -- Webflow Data API auth | +| `WEBFLOW_SITE_ID` | Var | New -- Target Webflow site ID | + +Default secrets/vars in the workflow: `TEST_WEBFLOW_API_TOKEN` / `TEST_WEBFLOW_SITE_ID`. + +### Webflow API Token Setup + +Generate at [webflow.com/dashboard/account/integrations](https://webflow.com/dashboard/account/integrations) > "Generate API Token". Required permissions: CMS read/write, Assets read/write. Store as a GitHub Actions secret. + +### Webflow Collection IDs + +The script discovers collection IDs dynamically by fetching all collections for the site and matching by slug: + +- **Categories collection slug:** `models-categories` +- **Models collection slug:** `models` + +This avoids hardcoding IDs that differ between test and production sites. + +## Data Flow + +### Categories Sync (runs first) + +1. Collect all unique modalities from API response (e.g., `["LLM", "Vision", "Audio", "Image"]`) +2. Fetch existing categories from Webflow: `GET /collections/{categories_collection_id}/items` +3. Match by slug (lowercase modality name, e.g., `"LLM"` -> `"llm"`, `"Vision"` -> `"vision"`) +4. Create any missing categories (never delete existing ones) +5. Build a `slug -> itemId` lookup map for use in model sync + +Category slugs are derived by lowercasing the modality name. Current modalities are single words (`LLM`, `Vision`, `Audio`, `Image`). If a multi-word modality appears in the future (e.g., `Text-to-Image`), Webflow's auto-slug on create will handle hyphenation, and subsequent syncs will match by the Webflow-assigned slug. + +### Logo Resolution + +For each model's `logo_url`: + +- **Regular URL** (starts with `http`): Pass directly to Image field as `{url: "...", alt: "{display_name} logo"}` +- **Base64 data URI** (starts with `data:`): Upload via Webflow Assets API (see below), use returned Webflow CDN URL in Image field +- **Null/empty**: Skip, no image + +#### Webflow Assets API Upload (two-step process) + +For base64 data URI logos: + +1. Decode the data URI to get the MIME type and binary buffer +2. Compute the MD5 hash of the file buffer (required by Webflow) +3. **Create asset metadata:** `POST https://api.webflow.com/v2/sites/{site_id}/assets` + ```json + { + "fileName": "{model_name}.{ext}", + "fileHash": "{md5_hex_digest}" + } + ``` + Response includes `uploadUrl` and `uploadDetails` (S3 presigned URL + form fields) +4. **Upload to S3:** `POST` (multipart/form-data) to the `uploadUrl` with all fields from `uploadDetails` plus the file buffer as the `file` field +5. The asset is now hosted on Webflow's CDN. Use the URL from the asset metadata response in the Image field. + +#### Image Diff Strategy + +When comparing existing Webflow items for updates, **skip the `logo` Image field in the diff comparison**. Instead, always set the logo on create, and on update only re-upload if the source `logo_url` value has changed (compare the original `logo_url` string from the API against a stored value, not the Webflow CDN URL). For the initial implementation, always include the logo URL on updates -- Webflow will skip re-downloading if the source URL hasn't changed. + +### Models Sync + +1. Transform API models (existing `transformModel` logic, minus `pricing`) +2. Fetch all existing model items from Webflow: `GET /collections/{models_collection_id}/items` (paginate if >100 items) +3. Match by slug (derived from `model.name`) +4. Diff: + - Not in Webflow -> add to **create** list + - In Webflow but data differs -> add to **update** list + - In Webflow and identical -> skip +5. Webflow items not in API response -> add to **delete** list +6. Execute batch create, batch update, batch delete (max 100 items per batch request; chunk if needed) +7. Publish all created and updated items (not deleted -- deletion removes them automatically) + +### Slug Handling + +Model slugs come from `model.name` in the API (e.g., `deepseek-v3-0324`, `flux2`, `glm-5`). These are already lowercase, hyphenated, alphanumeric strings that conform to Webflow's slug requirements. The script uses `model.name` directly as the Webflow slug without further normalization. + +### Batch Size Limits + +The Webflow v2 bulk CMS endpoints accept a maximum of **100 items per request**. With ~35 models currently, this fits in a single batch. If the model count grows beyond 100, the script must chunk operations into multiple batch requests. + +### Field Mapping + +| models.json field | Webflow CMS slug | CMS type | Transform | +|---|---|---|---| +| `name` | `slug` + `name` | built-in | Item name and slug | +| `display_name` | `display-name` | PlainText | Direct | +| `model_id` | `model-id` | PlainText | Direct | +| `logo_url` | `logo` | **Image** | URL or Asset API upload; alt = `"{display_name} logo"` | +| `description` | `description` | RichText | Wrap in `` tags if present; skip if undefined | +| `provider` | `provider` | PlainText | Direct; may be undefined for some models | +| `modalities` | `categories` | MultiReference | Array of strings -> array of category item IDs via slug lookup | +| `context_window` | `context-window` | PlainText | Direct; may be undefined | +| `total_params` | `total-params` | PlainText | Direct; may be undefined | +| `active_params` | `active-params` | PlainText | Direct; may be undefined | +| `precision` | `precision` | PlainText | Direct; may be undefined | +| `model_url` | `model-url` | Link | Direct; may be undefined | +| `isLive` | `live` | Switch | Direct | +| `isNew` | `new` | Switch | Direct | +| `isTrending` | `trending` | Switch | Direct | +| `pricing` | -- | -- | Dropped, not synced | +| -- | `player-mp4` | Link | Not managed by script, manual only | + +### Field Comparison for Updates + +Compare all synced field values (except `logo` Image field) between the transformed API model and the existing Webflow item. If any field differs, include the item in the update batch. No partial updates -- send all fields on update. Undefined API values are treated as empty strings for comparison purposes. + +## Schema Migration (One-Time) + +Before the first sync run, change the `logo` field on the test site Models collection: + +1. Delete the existing `logo` Link field +2. Create a new `logo` Image field + +Existing test data will lose logo values; the sync will repopulate them. + +## Error Handling + +- Any Modular Cloud API or Webflow API failure causes `process.exit(1)` +- Each operation logs what it's doing: `Creating model: deepseek-r1`, `Uploading logo for: flux2`, etc. +- Summary logged at end: `Created: X, Updated: Y, Deleted: Z, Unchanged: W` +- GitHub Action shows as failed if any error occurs + +## Workflow Changes (`fetch-models.yml`) + +```yaml +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + environment: + description: 'Target environment' + type: choice + options: + - test + - production + default: 'test' +``` + +The workflow uses the `environment` input to select which secrets/vars to use: + +- **test** (default): `TEST_WEBFLOW_API_TOKEN` (secret), `TEST_WEBFLOW_SITE_ID` (var) +- **production**: `WEBFLOW_API_TOKEN` (secret), `WEBFLOW_SITE_ID` (var) -- to be configured later + +The script receives `WEBFLOW_API_TOKEN` and `WEBFLOW_SITE_ID` as env vars regardless of which environment is selected; the workflow maps the appropriate secret/var names. + +Other changes: +- Remove the `git add` / `git commit` / `git push` step for `data/models.json` and `data/images/` +- Keep the existing Modular Cloud env vars + +## What Gets Removed + +- `data/models.json` file write +- `data/images/` directory and all committed image files +- `JSDELIVR_BASE` constant +- `MIME_TO_EXT` constant +- `parseDataUri` function (replaced by simpler base64 detection + Asset API upload) +- `processModelGarden` image file I/O loop +- `fs` imports (`writeFileSync`, `mkdirSync`, `rmSync`) +- Git commit step in the GitHub Actions workflow + +## What Gets Added + +- Webflow Data API client (fetch-based, no external dependencies) +- Categories sync logic +- Models diff + batch sync logic +- Logo resolution with Asset API upload for base64 images +- Collection ID discovery (fetch collections by site ID, find by slug) + +## Out of Scope + +- Production site sync (explicitly excluded, test site only for now) +- `player-mp4` field management (manual via page template) +- `pricing` field (dropped from sync) +- Webflow Designer API usage (Data API only) +- Rate limit retry logic (expected call volume is well below limits with ~35 models) +- Dry-run mode (may be added later) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index fe028f3..a04dc0a 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -1,35 +1,145 @@ -import { writeFileSync, mkdirSync, rmSync } from 'fs'; -import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; +import { createClient } from './webflow-api.js'; -const __dirname = dirname(fileURLToPath(import.meta.url)); +const __filename = fileURLToPath(import.meta.url); +const isMain = process.argv[1] === __filename; + +// -- Configuration -- const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; +const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID, DRY_RUN } = process.env; +const dryRun = DRY_RUN === 'true'; -if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { - console.error('Missing required environment variables: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL'); - process.exit(1); +let wf; +if (isMain) { + if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { + console.error( + 'Missing required env vars: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL' + ); + process.exit(1); + } + if (!WEBFLOW_API_TOKEN || !WEBFLOW_SITE_ID) { + console.error('Missing required env vars: WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID'); + process.exit(1); + } + wf = createClient(WEBFLOW_API_TOKEN); } -const JSDELIVR_BASE = 'https://cdn.jsdelivr.net/gh/modularml/modular-webflow@master/data/images'; - -const headers = { +const modularHeaders = { 'X-Yatai-Api-Token': MODULAR_CLOUD_API_TOKEN, 'X-Yatai-Organization': MODULAR_CLOUD_ORG, }; +// -- Modular Cloud API -- + async function fetchModelGarden() { - const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { headers }); + const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { + headers: modularHeaders, + }); if (!countRes.ok) throw new Error(`Count request failed: ${countRes.status}`); const { total } = await countRes.json(); const listRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden?count=${total}`, { - headers, + headers: modularHeaders, }); if (!listRes.ok) throw new Error(`List request failed: ${listRes.status}`); return listRes.json(); } +function normalizeApiModel(model) { + const meta = model.metadata || {}; + const tags = meta.tags || []; + return { + display_name: model.display_name, + name: model.name, + description: model.description, + model_id: model.model_id, + logo_url: meta.logo_url, + provider: meta.provider, + modalities: meta.modalities, + context_window: meta.context_window, + total_params: meta.total_params, + active_params: meta.active_params, + precision: meta.precision, + model_url: meta.model_url, + isLive: Boolean(model.gateway_id), + isNew: tags.includes('New'), + isTrending: tags.includes('Trending'), + }; +} + +// -- Webflow field mapping -- + +export function buildWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + logo: logoField, + description: model.description || '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + modalities: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} + +// -- Diff -- + +// Logo is resolved separately (upload vs URL); slug is identity, not content +const FIELDS_MANAGED_OUTSIDE_DIFF = new Set(['logo', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (FIELDS_MANAGED_OUTSIDE_DIFF.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + if ((apiVal === '' || apiVal == null) && (wfVal === '' || wfVal == null)) return false; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} + +// -- Logo resolution -- + const MIME_TO_EXT = { 'image/png': '.png', 'image/svg+xml': '.svg', @@ -39,9 +149,19 @@ const MIME_TO_EXT = { 'image/webp': '.webp', }; -function parseDataUri(dataUri) { - if (!dataUri || !dataUri.startsWith('data:')) return null; +function isUrl(str) { + return str.startsWith('http'); +} + +function isBase64DataUri(str) { + return str.startsWith('data:'); +} +function buildLogoField(model) { + return { url: model.logo_url, alt: `${model.display_name || model.name} logo` }; +} + +function parseBase64DataUri(dataUri) { const match = dataUri.match(/^data:([^;]+);base64,(.+)$/s); if (!match) return null; @@ -52,70 +172,194 @@ function parseDataUri(dataUri) { const buffer = Buffer.from(payload, 'base64'); if (buffer.length === 0) return null; - return { mime, ext, buffer }; + return { ext, buffer }; } -function transformModel(model) { - const meta = model.metadata || {}; - const tags = meta.tags || []; +async function uploadBase64Logo(model) { + const parsed = parseBase64DataUri(model.logo_url); + if (!parsed) return null; + + console.log(`Uploading logo for: ${model.name}`); + const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${model.name}${parsed.ext}`, parsed.buffer); + return { url: assetUrl, alt: `${model.display_name || model.name} logo` }; +} + +async function resolveLogo(model) { + const { logo_url } = model; + if (!logo_url) return null; + if (isUrl(logo_url)) return buildLogoField(model); + if (isBase64DataUri(logo_url)) return uploadBase64Logo(model); + return null; +} +// -- Orchestration helpers -- + +async function fetchModels() { + console.log('Fetching models from Modular Cloud API...'); + const modelGarden = await fetchModelGarden(); + const models = modelGarden.items.map(normalizeApiModel); + console.log(`Fetched ${models.length} models`); + return models; +} + +async function discoverCollections() { + const collections = await wf.getCollections(WEBFLOW_SITE_ID); return { - display_name: model.display_name, - name: model.name, - description: model.description, - model_id: model.model_id, - logo_url: meta.logo_url, - provider: meta.provider, - modalities: meta.modalities, - context_window: meta.context_window, - total_params: meta.total_params, - active_params: meta.active_params, - precision: meta.precision, - model_url: meta.model_url, - pricing: model.pricing, - isLive: Boolean(model.gateway_id), - isNew: tags.includes('New'), - isTrending: tags.includes('Trending'), + categoriesCol: wf.findCollectionBySlug(collections, 'models-category'), + modelsCol: wf.findCollectionBySlug(collections, 'models'), }; } -async function processModelGarden(modelGarden) { - const results = []; - const imagesDir = join(__dirname, '..', 'data', 'images'); - rmSync(imagesDir, { recursive: true, force: true }); - mkdirSync(imagesDir, { recursive: true }); - - for (const model of modelGarden.items) { - const transformed = transformModel(model); - - const parsed = parseDataUri(transformed.logo_url); - if (parsed) { - try { - const filename = `${transformed.name}${parsed.ext}`; - writeFileSync(join(imagesDir, filename), parsed.buffer); - transformed.logo_url = `${JSDELIVR_BASE}/${filename}`; - console.log(`Saved image for ${transformed.name}: ${filename}`); - } catch (err) { - console.error(`Failed to save image for ${transformed.name}: ${err.message}`); - transformed.logo_url = null; - } +function collectUniqueModalities(models) { + const all = new Set(); + for (const model of models) { + if (model.modalities) { + for (const m of model.modalities) all.add(m); + } + } + return all; +} + +function buildExistingCategoryMap(items) { + const map = new Map(); + for (const item of items) { + map.set(item.fieldData.slug, item.id); + } + return map; +} + +async function syncCategories(models, categoriesCollectionId) { + console.log('Syncing categories...'); + + const needed = collectUniqueModalities(models); + const existingItems = await wf.listCollectionItems(categoriesCollectionId); + const existingBySlug = buildExistingCategoryMap(existingItems); + + const categoryMap = {}; + for (const modality of needed) { + const slug = modality.toLowerCase(); + if (existingBySlug.has(slug)) { + categoryMap[slug] = existingBySlug.get(slug); + } else if (dryRun) { + console.log(`[dry run] Would create category: ${modality}`); + categoryMap[slug] = `dry-run-${slug}`; + } else { + console.log(`Creating category: ${modality}`); + const result = await wf.createItems(categoriesCollectionId, [{ name: modality, slug }]); + categoryMap[slug] = result.items[0].id; + } + } + + console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); + return categoryMap; +} + +async function fetchExistingItems(collectionId) { + console.log('Fetching existing Webflow items...'); + return wf.listCollectionItems(collectionId); +} + +function indexBySlug(items) { + const map = new Map(); + for (const item of items) { + map.set(item.fieldData.slug, item); + } + return map; +} + +async function buildModelFieldData(models, categoryMap, existingItems) { + console.log('Resolving logos and building field data...'); + const existingBySlug = indexBySlug(existingItems); + const apiModels = []; + + for (const model of models) { + const existing = existingBySlug.get(model.name); + const existingLogo = existing?.fieldData?.logo; + const hasBase64Logo = model.logo_url && isBase64DataUri(model.logo_url); + + // Reuse existing logo for base64 sources (avoid re-upload every run) + let logoField; + if (hasBase64Logo && existingLogo) { + logoField = existingLogo; + } else if (dryRun && hasBase64Logo) { + console.log(`[dry run] Would upload logo for: ${model.name}`); + logoField = { url: 'dry-run-placeholder', alt: `${model.display_name || model.name} logo` }; + } else { + logoField = await resolveLogo(model); } - results.push(transformed); + const fields = buildWebflowFields(model, model.modalities || [], categoryMap, logoField); + apiModels.push({ slug: model.name, fields }); + } + + return apiModels; +} + +async function applyChanges(collectionId, { toCreate, toUpdate, toDelete }) { + if (toCreate.length > 0) { + console.log(`Creating ${toCreate.length} models...`); + for (const fields of toCreate) console.log(` Creating: ${fields.slug}`); + await wf.createItems(collectionId, toCreate); + } + + if (toUpdate.length > 0) { + console.log(`Updating ${toUpdate.length} models...`); + for (const item of toUpdate) console.log(` Updating: ${item.fieldData.slug}`); + await wf.updateItems(collectionId, toUpdate); } - return results; + if (toDelete.length > 0) { + console.log(`Deleting ${toDelete.length} models...`); + await wf.deleteItems(collectionId, toDelete); + } +} + +function logDryRunSummary({ toCreate, toUpdate, toDelete, unchanged }) { + if (toCreate.length > 0) { + console.log(`[dry run] Would create ${toCreate.length} models:`); + for (const fields of toCreate) console.log(` ${fields.slug}`); + } + if (toUpdate.length > 0) { + console.log(`[dry run] Would update ${toUpdate.length} models:`); + for (const item of toUpdate) console.log(` ${item.fieldData.slug}`); + } + if (toDelete.length > 0) { + console.log(`[dry run] Would delete ${toDelete.length} models`); + } + console.log( + `\n[dry run] Summary — Create: ${toCreate.length}, Update: ${toUpdate.length}, Delete: ${toDelete.length}, Unchanged: ${unchanged}` + ); +} + +function logSyncSummary({ toCreate, toUpdate, toDelete, unchanged }) { + console.log( + `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` + ); } -fetchModelGarden() - .then((data) => processModelGarden(data)) - .then((models) => { - const outDir = join(__dirname, '..', 'data'); - mkdirSync(outDir, { recursive: true }); - writeFileSync(join(outDir, 'models.json'), JSON.stringify(models, null, 2)); - console.log(`Wrote ${models.length} models to ${outDir}/models.json`); - }) - .catch((err) => { +// -- Main -- + +async function main() { + if (dryRun) console.log('=== DRY RUN MODE — no changes will be pushed to Webflow ===\n'); + + const models = await fetchModels(); + const { categoriesCol, modelsCol } = await discoverCollections(); + const categoryMap = await syncCategories(models, categoriesCol.id); + const existingItems = await fetchExistingItems(modelsCol.id); + const apiModels = await buildModelFieldData(models, categoryMap, existingItems); + const changes = diffModels(apiModels, existingItems); + + if (dryRun) { + logDryRunSummary(changes); + } else { + await applyChanges(modelsCol.id, changes); + logSyncSummary(changes); + } +} + +if (isMain) { + main().catch((err) => { console.error(err); process.exit(1); }); +} diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js new file mode 100644 index 0000000..faaa607 --- /dev/null +++ b/scripts/webflow-api.js @@ -0,0 +1,220 @@ +import { createHash } from 'node:crypto'; + +function chunk(array, size) { + const chunks = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; +} + +function createClient(apiToken) { + async function webflowFetch(path, options = {}) { + const url = `https://api.webflow.com/v2${path}`; + const method = options.method || 'GET'; + + const response = await fetch(url, { + ...options, + method, + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + ...(options.headers || {}), + }, + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error( + `Webflow API error: ${method} ${path} => ${response.status} ${response.statusText}\n${body}` + ); + } + + if (response.status === 204) { + return null; + } + + return response.json(); + } + + // Detect whether /items/live endpoints are available (they 404 on + // sites that have never been published). Cache the result per collection. + const liveSupported = new Map(); + + async function supportsLive(collectionId) { + if (liveSupported.has(collectionId)) return liveSupported.get(collectionId); + + try { + await webflowFetch( + `/collections/${collectionId}/items/live?limit=1` + ); + liveSupported.set(collectionId, true); + return true; + } catch (err) { + if (err.message.includes('404')) { + console.log('Live endpoints not available for this site, using staged + publish'); + liveSupported.set(collectionId, false); + return false; + } + throw err; + } + } + + async function getCollections(siteId) { + const data = await webflowFetch(`/sites/${siteId}/collections`); + return data.collections; + } + + function findCollectionBySlug(collections, slug) { + const found = collections.find((c) => c.slug === slug); + if (!found) { + throw new Error(`Collection with slug "${slug}" not found`); + } + return found; + } + + async function listCollectionItems(collectionId) { + const limit = 100; + let offset = 0; + let allItems = []; + + while (true) { + const data = await webflowFetch( + `/collections/${collectionId}/items?limit=${limit}&offset=${offset}` + ); + const items = data.items || []; + allItems = allItems.concat(items); + + const total = data.pagination?.total ?? allItems.length; + if (allItems.length >= total || items.length === 0) { + break; + } + offset += limit; + } + + return allItems; + } + + async function publishItems(collectionId, itemIds) { + const batches = chunk(itemIds, 100); + for (const batch of batches) { + try { + await webflowFetch(`/collections/${collectionId}/items/publish`, { + method: 'POST', + body: JSON.stringify({ itemIds: batch }), + }); + } catch (err) { + if (err.message.includes('404')) { + console.log('Publish endpoint not available — site may need to be published first'); + return; + } + throw err; + } + } + } + + async function createItems(collectionId, fieldDataArray) { + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; + const batches = chunk(fieldDataArray, 100); + const allCreated = []; + + for (const batch of batches) { + const body = { + items: batch.map((fieldData) => ({ fieldData })), + }; + const data = await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { + method: 'POST', + body: JSON.stringify(body), + }); + const created = data?.items || []; + allCreated.push(...created); + } + + if (!useLiveEndpoint && allCreated.length > 0) { + await publishItems(collectionId, allCreated.map((item) => item.id)); + } + + return { items: allCreated }; + } + + async function updateItems(collectionId, itemsArray) { + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; + const batches = chunk(itemsArray, 100); + const allUpdated = []; + + for (const batch of batches) { + const data = await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { + method: 'PATCH', + body: JSON.stringify({ items: batch }), + }); + const updated = data?.items || []; + allUpdated.push(...updated); + } + + if (!useLiveEndpoint && allUpdated.length > 0) { + await publishItems(collectionId, allUpdated.map((item) => item.id)); + } + + return { items: allUpdated }; + } + + async function deleteItems(collectionId, itemIds) { + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; + const batches = chunk(itemIds, 100); + + for (const batch of batches) { + await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { + method: 'DELETE', + body: JSON.stringify({ itemIds: batch }), + }); + } + } + + async function uploadAsset(siteId, fileName, fileBuffer) { + const fileHash = createHash('md5').update(fileBuffer).digest('hex'); + + const metadata = await webflowFetch(`/sites/${siteId}/assets`, { + method: 'POST', + body: JSON.stringify({ fileName, fileHash }), + }); + + const { uploadUrl, uploadDetails } = metadata; + + const formData = new FormData(); + for (const [key, value] of Object.entries(uploadDetails)) { + formData.append(key, value); + } + formData.append('file', new Blob([fileBuffer]), fileName); + + const uploadResponse = await fetch(uploadUrl, { + method: 'POST', + body: formData, + }); + + if (!uploadResponse.ok) { + const body = await uploadResponse.text(); + throw new Error( + `Asset upload failed: ${uploadResponse.status} ${uploadResponse.statusText}\n${body}` + ); + } + + // Webflow's response shape varies by API version — check all known fields + return metadata.hostedUrl || metadata.url || metadata.assetUrl; + } + + return { + webflowFetch, + getCollections, + findCollectionBySlug, + listCollectionItems, + createItems, + updateItems, + deleteItems, + uploadAsset, + }; +} + +export { createClient, chunk }; diff --git a/tests/fetch-models/diff.test.js b/tests/fetch-models/diff.test.js new file mode 100644 index 0000000..0dc85d7 --- /dev/null +++ b/tests/fetch-models/diff.test.js @@ -0,0 +1,99 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { diffModels } from '../../scripts/fetch-models.js'; + +function makeApiModel(slug, fields) { + return { slug, fields: { slug, ...fields } }; +} + +function makeWfItem(id, slug, fieldData) { + return { id, fieldData: { slug, ...fieldData } }; +} + +describe('diffModels', () => { + it('identifies items to create when new in API but not in Webflow', () => { + const apiModels = [makeApiModel('new-model', { name: 'New Model' })]; + const webflowItems = []; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toCreate.length, 1); + assert.deepEqual(result.toCreate[0], { slug: 'new-model', name: 'New Model' }); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.toDelete.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('identifies items to delete when in Webflow but not in API', () => { + const apiModels = []; + const webflowItems = [makeWfItem('wf-id-1', 'old-model', { name: 'Old Model' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toDelete.length, 1); + assert.equal(result.toDelete[0], 'wf-id-1'); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('identifies items to update when fields differ', () => { + const apiModels = [makeApiModel('existing-model', { name: 'Updated Name' })]; + const webflowItems = [makeWfItem('wf-id-2', 'existing-model', { name: 'Old Name' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toUpdate.length, 1); + assert.equal(result.toUpdate[0].id, 'wf-id-2'); + assert.deepEqual(result.toUpdate[0].fieldData, { slug: 'existing-model', name: 'Updated Name' }); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toDelete.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('counts unchanged items when fields are identical', () => { + const apiModels = [makeApiModel('same-model', { name: 'Same Name', provider: 'Acme' })]; + const webflowItems = [makeWfItem('wf-id-3', 'same-model', { name: 'Same Name', provider: 'Acme' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.toDelete.length, 0); + }); + + it('ignores logo field differences in comparison', () => { + const apiModels = [ + makeApiModel('logo-model', { + name: 'Logo Model', + 'logo': { fileId: 'new-file-id', url: 'https://cdn.example.com/new.png' }, + }), + ]; + const webflowItems = [ + makeWfItem('wf-id-4', 'logo-model', { + name: 'Logo Model', + 'logo': { fileId: 'old-file-id', url: 'https://cdn.example.com/old.png' }, + }), + ]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toUpdate.length, 0); + }); + + it('treats empty string and undefined as equivalent', () => { + const apiModels = [ + makeApiModel('sparse-model', { name: 'Sparse', description: '', provider: '' }), + ]; + const webflowItems = [ + makeWfItem('wf-id-5', 'sparse-model', { name: 'Sparse' }), + ]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toUpdate.length, 0); + }); +}); diff --git a/tests/fetch-models/transform.test.js b/tests/fetch-models/transform.test.js new file mode 100644 index 0000000..ede42d3 --- /dev/null +++ b/tests/fetch-models/transform.test.js @@ -0,0 +1,96 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { buildWebflowFields } from '../../scripts/fetch-models.js'; + +describe('buildWebflowFields', () => { + it('maps all fields correctly for a full model', () => { + const model = { + display_name: 'My Model', + name: 'my-model', + model_id: 'org/my-model', + description: 'A great model', + provider: 'Acme', + context_window: 8192, + total_params: '70B', + active_params: '7B', + precision: 'fp16', + model_url: 'https://example.com/model', + isLive: true, + isNew: false, + isTrending: true, + }; + const modalities = ['Text', 'Image']; + const categoryMap = { text: 'id-text', image: 'id-image' }; + const logoField = { fileId: 'abc', url: 'https://cdn.example.com/logo.png' }; + + const result = buildWebflowFields(model, modalities, categoryMap, logoField); + + assert.equal(result.name, 'My Model'); + assert.equal(result.slug, 'my-model'); + assert.equal(result['display-name'], 'My Model'); + assert.equal(result['model-id'], 'org/my-model'); + assert.deepEqual(result.logo, logoField); + assert.equal(result.description, 'A great model'); + assert.equal(result.provider, 'Acme'); + assert.equal(result['context-window'], 8192); + assert.equal(result['total-params'], '70B'); + assert.equal(result['active-params'], '7B'); + assert.equal(result.precision, 'fp16'); + assert.equal(result['model-url'], 'https://example.com/model'); + assert.equal(result.live, true); + assert.equal(result.new, false); + assert.equal(result.trending, true); + assert.deepEqual(result.modalities, ['id-text', 'id-image']); + }); + + it('handles undefined optional fields with empty string defaults', () => { + const model = { + display_name: 'Minimal Model', + name: 'minimal-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = buildWebflowFields(model, [], {}, null); + + assert.equal(result['model-id'], ''); + assert.equal(result.description, ''); + assert.equal(result.provider, ''); + assert.equal(result['context-window'], ''); + assert.equal(result['total-params'], ''); + assert.equal(result['active-params'], ''); + assert.equal(result.precision, ''); + assert.equal(result['model-url'], ''); + assert.deepEqual(result.modalities, []); + }); + + it('falls back to name when display_name is falsy', () => { + const model = { + display_name: '', + name: 'fallback-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = buildWebflowFields(model, [], {}, null); + + assert.equal(result.name, 'fallback-model'); + assert.equal(result['display-name'], ''); + }); + + it('drops unknown modalities not in categoryMap', () => { + const model = { + display_name: 'Multi Model', + name: 'multi-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const modalities = ['Text', 'Video', 'Audio']; + const categoryMap = { text: 'id-text', audio: 'id-audio' }; + + const result = buildWebflowFields(model, modalities, categoryMap, null); + + assert.deepEqual(result.modalities, ['id-text', 'id-audio']); + }); +});