From d3ed6584c0b153d8d034a603d074acb4fb8f3cf5 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Fri, 20 Mar 2026 16:04:42 -0500 Subject: [PATCH 01/20] Add Webflow CMS sync design spec Design for modifying fetch-models.js to sync model data directly to Webflow CMS instead of writing models.json. Covers batch diff sync, logo upload via Assets API, and configurable site targeting. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-03-20-webflow-cms-sync-design.md | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md diff --git a/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md new file mode 100644 index 0000000..a1023f1 --- /dev/null +++ b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md @@ -0,0 +1,176 @@ +# Webflow CMS Sync Design + +**Date:** 2026-03-20 +**Status:** Draft +**Scope:** Replace `models.json` file output with direct Webflow CMS sync via Data API + +## Summary + +Modify `scripts/fetch-models.js` to sync model data from the Modular Cloud Model Garden API directly into Webflow CMS collections, replacing the current `models.json` + `data/images/` + jsDelivr pipeline. The Webflow CMS becomes the sole output of the script. + +**Target site:** Test Site - Blog Integration (`696947140cab938ac6990602`). The production Modular site must never be touched by this script. + +## Decisions + +| Decision | Choice | Rationale | +|---|---|---| +| Output target | Webflow CMS only, no `models.json` | CMS is the source of truth | +| Auth | Webflow Data API token (env var) | No MCP dependency in CI | +| Category management | Hybrid — reuse existing, auto-create new, never delete | New modalities appear organically | +| Model deletion | Delete from Webflow if absent from API | CMS mirrors API exactly | +| Matching key | `slug` (derived from `model.name`) | Stable, unique, Webflow-native lookup | +| Sync strategy | Single-pass batch diff | Minimizes API calls (~5 per run for ~35 models) | +| Image handling | Upload base64 to Webflow Assets API; pass URLs directly to Image field | Eliminates jsDelivr/git image pipeline | +| Error handling | Fail hard (exit 1) on any error, log summary | Clear signal in GitHub Actions | +| Site configurability | Workflow inputs select which token/site pair to use | Supports future production cutover | + +## Architecture + +### Three-Phase Pipeline + +``` +Modular Cloud API ──fetch──> Transform ──diff──> Webflow CMS + │ + Resolve logos + (URL passthrough + or Asset API upload) +``` + +**Phase 1 — Fetch:** Pull all models from Modular Cloud API (existing logic, unchanged). + +**Phase 2 — Diff:** Fetch all existing Webflow CMS items (categories + models), compare against API data, produce three lists: to-create, to-update, to-delete. + +**Phase 3 — Sync:** Execute batch create/update/delete against Webflow, then publish all changed items. + +### Environment Variables + +| Variable | Type | Purpose | +|---|---|---| +| `MODULAR_CLOUD_API_TOKEN` | Secret | Existing — Model Garden API auth | +| `MODULAR_CLOUD_ORG` | Secret | Existing — Model Garden org | +| `MODULAR_CLOUD_BASE_URL` | Var | Existing — Model Garden endpoint | +| `WEBFLOW_API_TOKEN` | Secret | New — Webflow Data API auth | +| `WEBFLOW_SITE_ID` | Var | New — Target Webflow site ID | + +Default secrets/vars in the workflow: `TEST_WEBFLOW_API_TOKEN` / `TEST_WEBFLOW_SITE_ID`. + +### Webflow API Token Setup + +Generate at [webflow.com/dashboard/account/integrations](https://webflow.com/dashboard/account/integrations) > "Generate API Token". Required permissions: CMS read/write, Assets read/write. Store as a GitHub Actions secret. + +## Data Flow + +### Categories Sync (runs first) + +1. Collect all unique modalities from API response (e.g., `["LLM", "Vision", "Audio", "Image"]`) +2. Fetch existing categories from Webflow: `GET /collections/{categories_collection_id}/items` +3. Match by slug (lowercase modality name) +4. Create any missing categories (never delete existing ones) +5. Build a `slug -> itemId` lookup map for use in model sync + +### Logo Resolution + +For each model's `logo_url`: + +- **Regular URL** (starts with `http`): Pass directly to Image field as `{url: "...", alt: "{display_name} logo"}` +- **Base64 data URI** (starts with `data:`): Decode to buffer, upload via Webflow Assets API (`POST /sites/{site_id}/assets` to create metadata + presigned URL, then upload buffer), use returned Webflow CDN URL in Image field +- **Null/empty**: Skip, no image + +### Models Sync + +1. Transform API models (existing `transformModel` logic, minus `pricing`) +2. Fetch all existing model items from Webflow: `GET /collections/{models_collection_id}/items` +3. Match by slug (derived from `model.name`) +4. Diff: + - Not in Webflow -> add to **create** list + - In Webflow but data differs -> add to **update** list + - In Webflow and identical -> skip +5. Webflow items not in API response -> add to **delete** list +6. Execute batch create, batch update, batch delete +7. Publish all affected items + +### Field Mapping + +| models.json field | Webflow CMS slug | CMS type | Transform | +|---|---|---|---| +| `name` | `slug` + `name` | built-in | Item name and slug | +| `display_name` | `display-name` | PlainText | Direct | +| `model_id` | `model-id` | PlainText | Direct | +| `logo_url` | `logo` | **Image** | URL or Asset API upload; alt = `"{display_name} logo"` | +| `description` | `description` | RichText | Wrap in `

` tags | +| `provider` | `provider` | PlainText | Direct | +| `modalities` | `categories` | MultiReference | Array of strings -> array of category item IDs via slug lookup | +| `context_window` | `context-window` | PlainText | Direct | +| `total_params` | `total-params` | PlainText | Direct | +| `active_params` | `active-params` | PlainText | Direct | +| `precision` | `precision` | PlainText | Direct | +| `model_url` | `model-url` | Link | Direct | +| `isLive` | `live` | Switch | Direct | +| `isNew` | `new` | Switch | Direct | +| `isTrending` | `trending` | Switch | Direct | +| `pricing` | — | — | Dropped, not synced | +| — | `player-mp4` | Link | Not managed by script, manual only | + +### Field Comparison for Updates + +Compare all synced field values between the transformed API model and the existing Webflow item. If any field differs, include the item in the update batch. No partial updates — send all fields on update. + +## Schema Migration (One-Time) + +Before the first sync run, change the `logo` field on the test site Models collection: + +1. Delete the existing `logo` Link field +2. Create a new `logo` Image field + +Existing test data will lose logo values; the sync will repopulate them. + +## Error Handling + +- Any Modular Cloud API or Webflow API failure causes `process.exit(1)` +- Each operation logs what it's doing: `Creating model: deepseek-r1`, `Uploading logo for: flux2`, etc. +- Summary logged at end: `Created: X, Updated: Y, Deleted: Z, Unchanged: W` +- GitHub Action shows as failed if any error occurs + +## Workflow Changes (`fetch-models.yml`) + +```yaml +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + webflow_token_secret: + description: 'Name of the secret containing the Webflow API token' + default: 'TEST_WEBFLOW_API_TOKEN' + webflow_site_id_var: + description: 'Name of the variable containing the Webflow site ID' + default: 'TEST_WEBFLOW_SITE_ID' +``` + +- Pass `WEBFLOW_API_TOKEN` and `WEBFLOW_SITE_ID` as env vars to the script +- Remove the `git add` / `git commit` step for `data/models.json` and `data/images/` +- Keep the existing Modular Cloud env vars + +## What Gets Removed + +- `data/models.json` file write +- `data/images/` directory and all committed image files +- `JSDELIVR_BASE` constant +- `parseDataUri` function (replaced by simpler base64 detection + Asset API upload) +- `processModelGarden` image file I/O loop +- Git commit step in the GitHub Actions workflow + +## What Gets Added + +- Webflow Data API client (fetch-based, no external dependencies) +- Categories sync logic +- Models diff + batch sync logic +- Logo resolution with Asset API upload for base64 images +- Collection ID discovery (fetch collections by site ID, find by slug) + +## Out of Scope + +- Production site sync (explicitly excluded, test site only) +- `player-mp4` field management (manual via page template) +- `pricing` field (dropped from sync) +- Webflow Designer API usage (Data API only) From 9500e1f5f7d100600c88e7dfce05f8825f96fc2a Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Fri, 20 Mar 2026 16:09:36 -0500 Subject: [PATCH 02/20] Update spec: address review findings - Fix MODULAR_CLOUD_ORG type (Var, not Secret) - Expand Asset API upload to full two-step flow with request shapes - Add batch size limits (100 items max) and chunking note - Clarify publish scope (created + updated only, not deleted) - Add collection slug discovery (models-categories, models) - Define slug handling for models and categories - Fix workflow config: use environment choice input instead of broken dynamic secret lookup - Add image diff strategy (skip logo in comparison) - Note undefined field handling - Add MIME_TO_EXT to removal list Co-Authored-By: Claude Opus 4.6 (1M context) --- .../2026-03-20-webflow-cms-sync-design.md | 124 +++++++++++++----- 1 file changed, 88 insertions(+), 36 deletions(-) diff --git a/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md index a1023f1..df968be 100644 --- a/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md +++ b/docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md @@ -16,41 +16,41 @@ Modify `scripts/fetch-models.js` to sync model data from the Modular Cloud Model |---|---|---| | Output target | Webflow CMS only, no `models.json` | CMS is the source of truth | | Auth | Webflow Data API token (env var) | No MCP dependency in CI | -| Category management | Hybrid — reuse existing, auto-create new, never delete | New modalities appear organically | +| Category management | Hybrid -- reuse existing, auto-create new, never delete | New modalities appear organically | | Model deletion | Delete from Webflow if absent from API | CMS mirrors API exactly | | Matching key | `slug` (derived from `model.name`) | Stable, unique, Webflow-native lookup | | Sync strategy | Single-pass batch diff | Minimizes API calls (~5 per run for ~35 models) | | Image handling | Upload base64 to Webflow Assets API; pass URLs directly to Image field | Eliminates jsDelivr/git image pipeline | | Error handling | Fail hard (exit 1) on any error, log summary | Clear signal in GitHub Actions | -| Site configurability | Workflow inputs select which token/site pair to use | Supports future production cutover | +| Site configurability | Workflow `environment` input selects test vs production config | Supports future production cutover | ## Architecture ### Three-Phase Pipeline ``` -Modular Cloud API ──fetch──> Transform ──diff──> Webflow CMS - │ +Modular Cloud API --fetch--> Transform --diff--> Webflow CMS + | Resolve logos (URL passthrough or Asset API upload) ``` -**Phase 1 — Fetch:** Pull all models from Modular Cloud API (existing logic, unchanged). +**Phase 1 -- Fetch:** Pull all models from Modular Cloud API (existing logic, unchanged). -**Phase 2 — Diff:** Fetch all existing Webflow CMS items (categories + models), compare against API data, produce three lists: to-create, to-update, to-delete. +**Phase 2 -- Diff:** Fetch all existing Webflow CMS items (categories + models), compare against API data, produce three lists: to-create, to-update, to-delete. -**Phase 3 — Sync:** Execute batch create/update/delete against Webflow, then publish all changed items. +**Phase 3 -- Sync:** Execute batch create/update/delete against Webflow, then publish created and updated items. ### Environment Variables | Variable | Type | Purpose | |---|---|---| -| `MODULAR_CLOUD_API_TOKEN` | Secret | Existing — Model Garden API auth | -| `MODULAR_CLOUD_ORG` | Secret | Existing — Model Garden org | -| `MODULAR_CLOUD_BASE_URL` | Var | Existing — Model Garden endpoint | -| `WEBFLOW_API_TOKEN` | Secret | New — Webflow Data API auth | -| `WEBFLOW_SITE_ID` | Var | New — Target Webflow site ID | +| `MODULAR_CLOUD_API_TOKEN` | Secret | Existing -- Model Garden API auth | +| `MODULAR_CLOUD_ORG` | Var | Existing -- Model Garden org | +| `MODULAR_CLOUD_BASE_URL` | Var | Existing -- Model Garden endpoint | +| `WEBFLOW_API_TOKEN` | Secret | New -- Webflow Data API auth | +| `WEBFLOW_SITE_ID` | Var | New -- Target Webflow site ID | Default secrets/vars in the workflow: `TEST_WEBFLOW_API_TOKEN` / `TEST_WEBFLOW_SITE_ID`. @@ -58,36 +58,76 @@ Default secrets/vars in the workflow: `TEST_WEBFLOW_API_TOKEN` / `TEST_WEBFLOW_S Generate at [webflow.com/dashboard/account/integrations](https://webflow.com/dashboard/account/integrations) > "Generate API Token". Required permissions: CMS read/write, Assets read/write. Store as a GitHub Actions secret. +### Webflow Collection IDs + +The script discovers collection IDs dynamically by fetching all collections for the site and matching by slug: + +- **Categories collection slug:** `models-categories` +- **Models collection slug:** `models` + +This avoids hardcoding IDs that differ between test and production sites. + ## Data Flow ### Categories Sync (runs first) 1. Collect all unique modalities from API response (e.g., `["LLM", "Vision", "Audio", "Image"]`) 2. Fetch existing categories from Webflow: `GET /collections/{categories_collection_id}/items` -3. Match by slug (lowercase modality name) +3. Match by slug (lowercase modality name, e.g., `"LLM"` -> `"llm"`, `"Vision"` -> `"vision"`) 4. Create any missing categories (never delete existing ones) 5. Build a `slug -> itemId` lookup map for use in model sync +Category slugs are derived by lowercasing the modality name. Current modalities are single words (`LLM`, `Vision`, `Audio`, `Image`). If a multi-word modality appears in the future (e.g., `Text-to-Image`), Webflow's auto-slug on create will handle hyphenation, and subsequent syncs will match by the Webflow-assigned slug. + ### Logo Resolution For each model's `logo_url`: - **Regular URL** (starts with `http`): Pass directly to Image field as `{url: "...", alt: "{display_name} logo"}` -- **Base64 data URI** (starts with `data:`): Decode to buffer, upload via Webflow Assets API (`POST /sites/{site_id}/assets` to create metadata + presigned URL, then upload buffer), use returned Webflow CDN URL in Image field +- **Base64 data URI** (starts with `data:`): Upload via Webflow Assets API (see below), use returned Webflow CDN URL in Image field - **Null/empty**: Skip, no image +#### Webflow Assets API Upload (two-step process) + +For base64 data URI logos: + +1. Decode the data URI to get the MIME type and binary buffer +2. Compute the MD5 hash of the file buffer (required by Webflow) +3. **Create asset metadata:** `POST https://api.webflow.com/v2/sites/{site_id}/assets` + ```json + { + "fileName": "{model_name}.{ext}", + "fileHash": "{md5_hex_digest}" + } + ``` + Response includes `uploadUrl` and `uploadDetails` (S3 presigned URL + form fields) +4. **Upload to S3:** `POST` (multipart/form-data) to the `uploadUrl` with all fields from `uploadDetails` plus the file buffer as the `file` field +5. The asset is now hosted on Webflow's CDN. Use the URL from the asset metadata response in the Image field. + +#### Image Diff Strategy + +When comparing existing Webflow items for updates, **skip the `logo` Image field in the diff comparison**. Instead, always set the logo on create, and on update only re-upload if the source `logo_url` value has changed (compare the original `logo_url` string from the API against a stored value, not the Webflow CDN URL). For the initial implementation, always include the logo URL on updates -- Webflow will skip re-downloading if the source URL hasn't changed. + ### Models Sync 1. Transform API models (existing `transformModel` logic, minus `pricing`) -2. Fetch all existing model items from Webflow: `GET /collections/{models_collection_id}/items` +2. Fetch all existing model items from Webflow: `GET /collections/{models_collection_id}/items` (paginate if >100 items) 3. Match by slug (derived from `model.name`) 4. Diff: - Not in Webflow -> add to **create** list - In Webflow but data differs -> add to **update** list - In Webflow and identical -> skip 5. Webflow items not in API response -> add to **delete** list -6. Execute batch create, batch update, batch delete -7. Publish all affected items +6. Execute batch create, batch update, batch delete (max 100 items per batch request; chunk if needed) +7. Publish all created and updated items (not deleted -- deletion removes them automatically) + +### Slug Handling + +Model slugs come from `model.name` in the API (e.g., `deepseek-v3-0324`, `flux2`, `glm-5`). These are already lowercase, hyphenated, alphanumeric strings that conform to Webflow's slug requirements. The script uses `model.name` directly as the Webflow slug without further normalization. + +### Batch Size Limits + +The Webflow v2 bulk CMS endpoints accept a maximum of **100 items per request**. With ~35 models currently, this fits in a single batch. If the model count grows beyond 100, the script must chunk operations into multiple batch requests. ### Field Mapping @@ -97,23 +137,23 @@ For each model's `logo_url`: | `display_name` | `display-name` | PlainText | Direct | | `model_id` | `model-id` | PlainText | Direct | | `logo_url` | `logo` | **Image** | URL or Asset API upload; alt = `"{display_name} logo"` | -| `description` | `description` | RichText | Wrap in `

` tags | -| `provider` | `provider` | PlainText | Direct | +| `description` | `description` | RichText | Wrap in `

` tags if present; skip if undefined | +| `provider` | `provider` | PlainText | Direct; may be undefined for some models | | `modalities` | `categories` | MultiReference | Array of strings -> array of category item IDs via slug lookup | -| `context_window` | `context-window` | PlainText | Direct | -| `total_params` | `total-params` | PlainText | Direct | -| `active_params` | `active-params` | PlainText | Direct | -| `precision` | `precision` | PlainText | Direct | -| `model_url` | `model-url` | Link | Direct | +| `context_window` | `context-window` | PlainText | Direct; may be undefined | +| `total_params` | `total-params` | PlainText | Direct; may be undefined | +| `active_params` | `active-params` | PlainText | Direct; may be undefined | +| `precision` | `precision` | PlainText | Direct; may be undefined | +| `model_url` | `model-url` | Link | Direct; may be undefined | | `isLive` | `live` | Switch | Direct | | `isNew` | `new` | Switch | Direct | | `isTrending` | `trending` | Switch | Direct | -| `pricing` | — | — | Dropped, not synced | -| — | `player-mp4` | Link | Not managed by script, manual only | +| `pricing` | -- | -- | Dropped, not synced | +| -- | `player-mp4` | Link | Not managed by script, manual only | ### Field Comparison for Updates -Compare all synced field values between the transformed API model and the existing Webflow item. If any field differs, include the item in the update batch. No partial updates — send all fields on update. +Compare all synced field values (except `logo` Image field) between the transformed API model and the existing Webflow item. If any field differs, include the item in the update batch. No partial updates -- send all fields on update. Undefined API values are treated as empty strings for comparison purposes. ## Schema Migration (One-Time) @@ -139,16 +179,24 @@ on: - cron: '0 0 * * *' workflow_dispatch: inputs: - webflow_token_secret: - description: 'Name of the secret containing the Webflow API token' - default: 'TEST_WEBFLOW_API_TOKEN' - webflow_site_id_var: - description: 'Name of the variable containing the Webflow site ID' - default: 'TEST_WEBFLOW_SITE_ID' + environment: + description: 'Target environment' + type: choice + options: + - test + - production + default: 'test' ``` -- Pass `WEBFLOW_API_TOKEN` and `WEBFLOW_SITE_ID` as env vars to the script -- Remove the `git add` / `git commit` step for `data/models.json` and `data/images/` +The workflow uses the `environment` input to select which secrets/vars to use: + +- **test** (default): `TEST_WEBFLOW_API_TOKEN` (secret), `TEST_WEBFLOW_SITE_ID` (var) +- **production**: `WEBFLOW_API_TOKEN` (secret), `WEBFLOW_SITE_ID` (var) -- to be configured later + +The script receives `WEBFLOW_API_TOKEN` and `WEBFLOW_SITE_ID` as env vars regardless of which environment is selected; the workflow maps the appropriate secret/var names. + +Other changes: +- Remove the `git add` / `git commit` / `git push` step for `data/models.json` and `data/images/` - Keep the existing Modular Cloud env vars ## What Gets Removed @@ -156,8 +204,10 @@ on: - `data/models.json` file write - `data/images/` directory and all committed image files - `JSDELIVR_BASE` constant +- `MIME_TO_EXT` constant - `parseDataUri` function (replaced by simpler base64 detection + Asset API upload) - `processModelGarden` image file I/O loop +- `fs` imports (`writeFileSync`, `mkdirSync`, `rmSync`) - Git commit step in the GitHub Actions workflow ## What Gets Added @@ -170,7 +220,9 @@ on: ## Out of Scope -- Production site sync (explicitly excluded, test site only) +- Production site sync (explicitly excluded, test site only for now) - `player-mp4` field management (manual via page template) - `pricing` field (dropped from sync) - Webflow Designer API usage (Data API only) +- Rate limit retry logic (expected call volume is well below limits with ~35 models) +- Dry-run mode (may be added later) From 9c007450cc3a83b9ed667a5403e3309a4e8cba47 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:12:13 -0500 Subject: [PATCH 03/20] Add Webflow CMS sync implementation plan 7 tasks covering: schema migration, Webflow API client, transform/diff logic with tests, main script rewrite, workflow update, cleanup, and integration testing. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plans/2026-03-23-webflow-cms-sync.md | 926 ++++++++++++++++++ 1 file changed, 926 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-23-webflow-cms-sync.md diff --git a/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md b/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md new file mode 100644 index 0000000..107bad6 --- /dev/null +++ b/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md @@ -0,0 +1,926 @@ +# Webflow CMS Sync Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace `models.json` file output in `fetch-models.js` with direct Webflow CMS sync via the Data API. + +**Architecture:** The script becomes a three-phase pipeline: fetch from Modular Cloud API, diff against existing Webflow CMS state, then batch sync (create/update/delete). A separate `webflow-api.js` module handles all Webflow Data API calls. Logo images are either passed as URLs or uploaded via the Assets API. + +**Tech Stack:** Node.js 20 (ES modules), Webflow Data API v2, `node:test` for unit tests, `node:crypto` for MD5 hashing. + +**Spec:** `docs/superpowers/specs/2026-03-20-webflow-cms-sync-design.md` + +**CRITICAL: NEVER target the production Webflow site (`68c9c3107effc2ea46e1a81f`). Only the test site (`696947140cab938ac6990602`).** + +--- + +## File Structure + +| File | Action | Responsibility | +|---|---|---| +| `scripts/webflow-api.js` | Create | Webflow Data API client: collections, items CRUD, assets upload, publish | +| `scripts/fetch-models.js` | Rewrite | Orchestration: fetch API, transform, diff, sync categories, resolve logos, sync models | +| `.github/workflows/fetch-models.yml` | Modify | Add environment input, Webflow env vars, remove git commit step | +| `tests/fetch-models/diff.test.js` | Create | Unit tests for diff logic (pure functions) | +| `tests/fetch-models/transform.test.js` | Create | Unit tests for field mapping and transform | +| `data/models.json` | Delete | No longer needed | +| `data/images/` | Delete | No longer needed | + +--- + +### Task 0: Schema Migration + +One-time change to the test site CMS: convert the `logo` field from Link to Image type. + +**Files:** +- No code files; this is a Webflow API operation + +- [ ] **Step 1: Delete the existing `logo` Link field from the Models collection** + +Use the Webflow MCP or Data API to delete the `logo` field (id: `c95daeedec88cf91cb55c7a5b182e2e4`) from collection `69bda3d2ff82137fe97f16d9`. + +- [ ] **Step 2: Create a new `logo` Image field on the Models collection** + +Create a static field with type `Image` and displayName `Logo` on collection `69bda3d2ff82137fe97f16d9`. + +- [ ] **Step 3: Verify the field exists** + +Fetch collection details for `69bda3d2ff82137fe97f16d9` and confirm a field with slug `logo` and type `ImageRef` (or `Image`) exists. + +--- + +### Task 1: Webflow API Client + +Build the low-level Webflow Data API wrapper. All Webflow HTTP calls go through this module. + +**Files:** +- Create: `scripts/webflow-api.js` + +- [ ] **Step 1: Create the module with base fetch helper** + +```js +// scripts/webflow-api.js +const WEBFLOW_API_BASE = 'https://api.webflow.com/v2'; + +function createClient(apiToken) { + async function webflowFetch(path, options = {}) { + const url = `${WEBFLOW_API_BASE}${path}`; + const res = await fetch(url, { + ...options, + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + ...options.headers, + }, + }); + if (!res.ok) { + const body = await res.text(); + throw new Error(`Webflow API ${options.method || 'GET'} ${path} failed (${res.status}): ${body}`); + } + if (res.status === 204) return null; + return res.json(); + } + + return { webflowFetch }; +} + +export { createClient }; +``` + +- [ ] **Step 2: Add collection discovery** + +```js +async function getCollections(siteId) { + const data = await webflowFetch(`/sites/${siteId}/collections`); + return data.collections; +} + +function findCollectionBySlug(collections, slug) { + const col = collections.find((c) => c.slug === slug); + if (!col) throw new Error(`Collection with slug "${slug}" not found`); + return col; +} +``` + +Add these inside `createClient` and export them on the returned object. + +- [ ] **Step 3: Add list items with pagination** + +```js +async function listCollectionItems(collectionId) { + const items = []; + let offset = 0; + const limit = 100; + while (true) { + const data = await webflowFetch( + `/collections/${collectionId}/items?limit=${limit}&offset=${offset}` + ); + items.push(...data.items); + if (items.length >= data.pagination.total) break; + offset += limit; + } + return items; +} +``` + +- [ ] **Step 4: Add batch create, update, delete, and publish** + +```js +async function createItems(collectionId, fieldDataArray) { + return webflowFetch(`/collections/${collectionId}/items`, { + method: 'POST', + body: JSON.stringify({ fieldData: fieldDataArray }), + }); +} + +async function updateItems(collectionId, itemsArray) { + return webflowFetch(`/collections/${collectionId}/items`, { + method: 'PATCH', + body: JSON.stringify({ items: itemsArray }), + }); +} + +async function deleteItems(collectionId, itemIds) { + return webflowFetch(`/collections/${collectionId}/items`, { + method: 'DELETE', + body: JSON.stringify({ items: itemIds.map((id) => ({ id })) }), + }); +} + +async function publishItems(collectionId, itemIds) { + return webflowFetch(`/collections/${collectionId}/items/publish`, { + method: 'POST', + body: JSON.stringify({ itemIds }), + }); +} +``` + +- [ ] **Step 5: Add asset upload (two-step)** + +```js +async function uploadAsset(siteId, fileName, fileBuffer) { + const crypto = await import('node:crypto'); + const fileHash = crypto.createHash('md5').update(fileBuffer).digest('hex'); + + // Step 1: Create asset metadata + const metadata = await webflowFetch(`/sites/${siteId}/assets`, { + method: 'POST', + body: JSON.stringify({ fileName, fileHash }), + }); + + // Step 2: Upload to S3 presigned URL + const form = new FormData(); + for (const [key, value] of Object.entries(metadata.uploadDetails)) { + form.append(key, value); + } + form.append('file', new Blob([fileBuffer]), fileName); + + const uploadRes = await fetch(metadata.uploadUrl, { + method: 'POST', + body: form, + }); + if (!uploadRes.ok) { + throw new Error(`Asset upload to S3 failed (${uploadRes.status})`); + } + + return metadata.assetUrl; +} +``` + +- [ ] **Step 6: Export all functions and commit** + +Ensure `createClient` returns all functions: `getCollections`, `findCollectionBySlug`, `listCollectionItems`, `createItems`, `updateItems`, `deleteItems`, `publishItems`, `uploadAsset`. + +```bash +git add scripts/webflow-api.js +git commit -m "feat: add Webflow Data API client module" +``` + +--- + +### Task 2: Transform and Diff Logic (with tests) + +Build the pure functions for transforming API data to Webflow field format and diffing against existing items. + +**Files:** +- Create: `tests/fetch-models/transform.test.js` +- Create: `tests/fetch-models/diff.test.js` +- Modify: `scripts/fetch-models.js` (add exported pure functions, keep existing code for now) + +- [ ] **Step 1: Write failing tests for `toWebflowFields`** + +```js +// tests/fetch-models/transform.test.js +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { toWebflowFields } from '../../scripts/fetch-models.js'; + +describe('toWebflowFields', () => { + it('maps all fields correctly', () => { + const model = { + display_name: 'DeepSeek V3', + name: 'deepseek-v3', + description: 'A great model.', + model_id: 'deepseek-ai/DeepSeek-V3', + provider: 'DeepSeek', + context_window: '128K', + total_params: '671B', + active_params: '37B', + precision: 'FP8', + model_url: 'https://huggingface.co/deepseek-ai/DeepSeek-V3', + isLive: true, + isNew: false, + isTrending: true, + }; + const categoryMap = { llm: 'cat-id-1', vision: 'cat-id-2' }; + const modalities = ['LLM', 'Vision']; + const logoField = { url: 'https://example.com/logo.png', alt: 'DeepSeek V3 logo' }; + + const result = toWebflowFields(model, modalities, categoryMap, logoField); + + assert.equal(result.name, 'DeepSeek V3'); + assert.equal(result.slug, 'deepseek-v3'); + assert.equal(result['display-name'], 'DeepSeek V3'); + assert.equal(result['model-id'], 'deepseek-ai/DeepSeek-V3'); + assert.equal(result.description, '

A great model.

'); + assert.equal(result.provider, 'DeepSeek'); + assert.equal(result['context-window'], '128K'); + assert.equal(result['total-params'], '671B'); + assert.equal(result['active-params'], '37B'); + assert.equal(result.precision, 'FP8'); + assert.equal(result['model-url'], 'https://huggingface.co/deepseek-ai/DeepSeek-V3'); + assert.equal(result.live, true); + assert.equal(result.new, false); + assert.equal(result.trending, true); + assert.deepEqual(result.categories, ['cat-id-1', 'cat-id-2']); + assert.deepEqual(result.logo, logoField); + }); + + it('handles undefined optional fields', () => { + const model = { + display_name: 'Test', + name: 'test', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result.provider, ''); + assert.equal(result.description, ''); + assert.equal(result['model-url'], ''); + assert.equal(result.logo, null); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `node --test tests/fetch-models/transform.test.js` +Expected: FAIL — `toWebflowFields` not found + +- [ ] **Step 3: Write `toWebflowFields` in fetch-models.js** + +Add at the top of `scripts/fetch-models.js` (keep existing code below for now): + +```js +export function toWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + logo: logoField, + description: model.description ? `

${model.description}

` : '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `node --test tests/fetch-models/transform.test.js` +Expected: PASS + +- [ ] **Step 5: Write failing tests for `diffModels`** + +```js +// tests/fetch-models/diff.test.js +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { diffModels } from '../../scripts/fetch-models.js'; + +describe('diffModels', () => { + it('identifies items to create', () => { + const apiModels = [{ slug: 'new-model', fields: { name: 'New' } }]; + const webflowItems = []; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 1); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 0); + }); + + it('identifies items to delete', () => { + const apiModels = []; + const webflowItems = [{ id: 'wf-1', fieldData: { slug: 'old-model', name: 'Old' } }]; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 1); + assert.equal(toDelete[0], 'wf-1'); + }); + + it('identifies items to update when fields differ', () => { + const apiModels = [{ slug: 'model-a', fields: { name: 'Model A', provider: 'New Co' } }]; + const webflowItems = [ + { id: 'wf-1', fieldData: { slug: 'model-a', name: 'Model A', provider: 'Old Co' } }, + ]; + const { toCreate, toUpdate, toDelete } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 1); + assert.equal(toUpdate[0].id, 'wf-1'); + assert.equal(toDelete.length, 0); + }); + + it('skips unchanged items', () => { + const fields = { name: 'Model A', provider: 'Same Co', live: true }; + const apiModels = [{ slug: 'model-a', fields }]; + const webflowItems = [{ id: 'wf-1', fieldData: { slug: 'model-a', ...fields } }]; + const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); + + assert.equal(toCreate.length, 0); + assert.equal(toUpdate.length, 0); + assert.equal(toDelete.length, 0); + assert.equal(unchanged, 1); + }); + + it('ignores logo field in comparison', () => { + const apiModels = [ + { slug: 'model-a', fields: { name: 'A', logo: { url: 'new.png', alt: 'A logo' } } }, + ]; + const webflowItems = [ + { + id: 'wf-1', + fieldData: { + slug: 'model-a', + name: 'A', + logo: { fileId: '123', url: 'https://cdn.webflow.com/old.png', alt: 'A logo' }, + }, + }, + ]; + const { toUpdate, unchanged } = diffModels(apiModels, webflowItems); + + assert.equal(toUpdate.length, 0); + assert.equal(unchanged, 1); + }); +}); +``` + +- [ ] **Step 6: Run tests to verify they fail** + +Run: `node --test tests/fetch-models/diff.test.js` +Expected: FAIL — `diffModels` not found + +- [ ] **Step 7: Write `diffModels`** + +Add to `scripts/fetch-models.js`: + +```js +const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (SKIP_DIFF_FIELDS.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} +``` + +- [ ] **Step 8: Run tests to verify they pass** + +Run: `node --test tests/fetch-models/diff.test.js` +Expected: PASS + +- [ ] **Step 9: Commit** + +```bash +git add tests/fetch-models/ scripts/fetch-models.js +git commit -m "feat: add toWebflowFields and diffModels with tests" +``` + +--- + +### Task 3: Rewrite fetch-models.js Main Script + +Replace the old orchestration with the new sync pipeline. Keep `fetchModelGarden` and `transformModel` (minus `pricing`), remove everything else. + +**Files:** +- Rewrite: `scripts/fetch-models.js` + +- [ ] **Step 1: Rewrite the full script** + +```js +// scripts/fetch-models.js +import { createHash } from 'node:crypto'; +import { createClient } from './webflow-api.js'; + +// -- Environment variables -- + +const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; +const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID } = process.env; + +if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { + console.error( + 'Missing required env vars: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL' + ); + process.exit(1); +} +if (!WEBFLOW_API_TOKEN || !WEBFLOW_SITE_ID) { + console.error('Missing required env vars: WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID'); + process.exit(1); +} + +const modularHeaders = { + 'X-Yatai-Api-Token': MODULAR_CLOUD_API_TOKEN, + 'X-Yatai-Organization': MODULAR_CLOUD_ORG, +}; + +const wf = createClient(WEBFLOW_API_TOKEN); + +// -- Modular Cloud API (unchanged) -- + +async function fetchModelGarden() { + const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { + headers: modularHeaders, + }); + if (!countRes.ok) throw new Error(`Count request failed: ${countRes.status}`); + const { total } = await countRes.json(); + + const listRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden?count=${total}`, { + headers: modularHeaders, + }); + if (!listRes.ok) throw new Error(`List request failed: ${listRes.status}`); + return listRes.json(); +} + +function transformModel(model) { + const meta = model.metadata || {}; + const tags = meta.tags || []; + return { + display_name: model.display_name, + name: model.name, + description: model.description, + model_id: model.model_id, + logo_url: meta.logo_url, + provider: meta.provider, + modalities: meta.modalities, + context_window: meta.context_window, + total_params: meta.total_params, + active_params: meta.active_params, + precision: meta.precision, + model_url: meta.model_url, + isLive: Boolean(model.gateway_id), + isNew: tags.includes('New'), + isTrending: tags.includes('Trending'), + }; +} + +// -- Field mapping -- + +export function toWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + logo: logoField, + description: model.description ? `

${model.description}

` : '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} + +// -- Diff -- + +const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (SKIP_DIFF_FIELDS.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} + +// -- Logo resolution -- + +const MIME_TO_EXT = { + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/gif': '.gif', + 'image/webp': '.webp', +}; + +async function resolveLogo(model) { + const { logo_url, display_name, name } = model; + + if (!logo_url) return null; + + if (logo_url.startsWith('http')) { + return { url: logo_url, alt: `${display_name || name} logo` }; + } + + if (logo_url.startsWith('data:')) { + const match = logo_url.match(/^data:([^;]+);base64,(.+)$/s); + if (!match) return null; + + const [, mime, payload] = match; + const ext = MIME_TO_EXT[mime]; + if (!ext) return null; + + const buffer = Buffer.from(payload, 'base64'); + if (buffer.length === 0) return null; + + console.log(`Uploading logo for: ${name}`); + const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${name}${ext}`, buffer); + return { url: assetUrl, alt: `${display_name || name} logo` }; + } + + return null; +} + +// -- Categories sync -- + +async function syncCategories(models, categoriesCollectionId) { + const allModalities = new Set(); + for (const model of models) { + if (model.modalities) { + for (const m of model.modalities) allModalities.add(m); + } + } + + const existingItems = await wf.listCollectionItems(categoriesCollectionId); + const existingBySlug = new Map(); + for (const item of existingItems) { + existingBySlug.set(item.fieldData.slug, item.id); + } + + const categoryMap = {}; + for (const modality of allModalities) { + const slug = modality.toLowerCase(); + if (existingBySlug.has(slug)) { + categoryMap[slug] = existingBySlug.get(slug); + } else { + console.log(`Creating category: ${modality}`); + const result = await wf.createItems(categoriesCollectionId, [ + { name: modality, slug }, + ]); + const newItem = result.items[0]; + categoryMap[slug] = newItem.id; + await wf.publishItems(categoriesCollectionId, [newItem.id]); + } + } + + return categoryMap; +} + +// -- Main -- + +async function main() { + // Phase 1: Fetch + console.log('Fetching models from Modular Cloud API...'); + const modelGarden = await fetchModelGarden(); + const models = modelGarden.items.map(transformModel); + console.log(`Fetched ${models.length} models`); + + // Discover collections + const collections = await wf.getCollections(WEBFLOW_SITE_ID); + const categoriesCol = wf.findCollectionBySlug(collections, 'models-categories'); + const modelsCol = wf.findCollectionBySlug(collections, 'models'); + + // Sync categories + console.log('Syncing categories...'); + const categoryMap = await syncCategories(models, categoriesCol.id); + console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); + + // Resolve logos and build field data + console.log('Resolving logos and building field data...'); + const apiModels = []; + for (const model of models) { + const logoField = await resolveLogo(model); + const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); + apiModels.push({ slug: model.name, fields }); + } + + // Phase 2: Diff + console.log('Fetching existing Webflow items...'); + const webflowItems = await wf.listCollectionItems(modelsCol.id); + const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); + + // Phase 3: Sync + const publishIds = []; + + if (toCreate.length > 0) { + console.log(`Creating ${toCreate.length} models...`); + for (const fields of toCreate) { + console.log(` Creating: ${fields.slug}`); + } + const created = await wf.createItems(modelsCol.id, toCreate); + publishIds.push(...created.items.map((item) => item.id)); + } + + if (toUpdate.length > 0) { + console.log(`Updating ${toUpdate.length} models...`); + for (const item of toUpdate) { + console.log(` Updating: ${item.fieldData.slug}`); + } + await wf.updateItems(modelsCol.id, toUpdate); + publishIds.push(...toUpdate.map((item) => item.id)); + } + + if (toDelete.length > 0) { + console.log(`Deleting ${toDelete.length} models...`); + await wf.deleteItems(modelsCol.id, toDelete); + } + + if (publishIds.length > 0) { + console.log(`Publishing ${publishIds.length} items...`); + await wf.publishItems(modelsCol.id, publishIds); + } + + // Summary + console.log( + `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` + ); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); +``` + +- [ ] **Step 2: Verify tests still pass** + +Run: `node --test tests/fetch-models/transform.test.js tests/fetch-models/diff.test.js` +Expected: PASS + +- [ ] **Step 3: Commit** + +```bash +git add scripts/fetch-models.js +git commit -m "feat: rewrite fetch-models.js for Webflow CMS sync" +``` + +--- + +### Task 4: Update GitHub Actions Workflow + +**Files:** +- Modify: `.github/workflows/fetch-models.yml` + +- [ ] **Step 1: Rewrite the workflow** + +```yaml +name: Fetch Model Garden + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + inputs: + environment: + description: 'Target environment' + type: choice + options: + - test + - production + default: 'test' + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Sync models to Webflow + run: node scripts/fetch-models.js + env: + MODULAR_CLOUD_API_TOKEN: ${{ secrets.MODULAR_CLOUD_API_TOKEN }} + MODULAR_CLOUD_ORG: ${{ vars.MODULAR_CLOUD_ORG }} + MODULAR_CLOUD_BASE_URL: ${{ vars.MODULAR_CLOUD_BASE_URL }} + WEBFLOW_API_TOKEN: ${{ inputs.environment == 'production' && secrets.WEBFLOW_API_TOKEN || secrets.TEST_WEBFLOW_API_TOKEN }} + WEBFLOW_SITE_ID: ${{ inputs.environment == 'production' && vars.WEBFLOW_SITE_ID || vars.TEST_WEBFLOW_SITE_ID }} +``` + +Key changes: +- Removed `permissions: contents: write` (no longer writing to repo) +- Removed the `git commit` / `git push` step entirely +- Added `workflow_dispatch` inputs with environment choice +- Added conditional env var selection for test vs production +- Renamed job from `fetch` to `sync` + +- [ ] **Step 2: Commit** + +```bash +git add .github/workflows/fetch-models.yml +git commit -m "feat: update workflow for Webflow CMS sync with environment selection" +``` + +--- + +### Task 5: Clean Up Old Files + +Remove files and directories that are no longer needed. + +**Files:** +- Delete: `data/models.json` +- Delete: `data/images/` (entire directory) + +- [ ] **Step 1: Remove old data files** + +```bash +rm data/models.json +rm -rf data/images/ +``` + +- [ ] **Step 2: Verify nothing references the removed files** + +Search the codebase for references to `models.json` or `data/images`: + +```bash +grep -r "models.json" --include="*.js" --include="*.ts" --include="*.yml" . +grep -r "data/images" --include="*.js" --include="*.ts" --include="*.yml" . +``` + +Expected: No results (or only references in the spec/plan docs). + +- [ ] **Step 3: Commit** + +```bash +git add -A data/ +git commit -m "chore: remove models.json and data/images (replaced by Webflow CMS)" +``` + +--- + +### Task 6: Integration Test Against Test Site + +Run the script against the real Webflow test site to verify end-to-end functionality. + +**Files:** +- No new files; uses existing script + +- [ ] **Step 1: Run the schema migration (Task 0) if not done yet** + +Ensure the `logo` field on the test site Models collection is type Image, not Link. + +- [ ] **Step 2: Set environment variables locally** + +```bash +export MODULAR_CLOUD_API_TOKEN="..." +export MODULAR_CLOUD_ORG="..." +export MODULAR_CLOUD_BASE_URL="..." +export WEBFLOW_API_TOKEN="..." # Test site token +export WEBFLOW_SITE_ID="696947140cab938ac6990602" # Test site only! +``` + +- [ ] **Step 3: Run the script** + +```bash +node scripts/fetch-models.js +``` + +Expected output: +``` +Fetching models from Modular Cloud API... +Fetched N models +Syncing categories... +Categories ready: llm, vision, audio, image +Resolving logos and building field data... +Fetching existing Webflow items... +Creating X models... +Updating Y models... +Deleting Z models... +Publishing N items... + +Sync complete. Created: X, Updated: Y, Deleted: Z, Unchanged: W +``` + +- [ ] **Step 4: Verify in Webflow** + +Check the test site CMS in the Webflow dashboard: +- Categories collection has the expected items +- Models collection has the expected items with all fields populated +- Logo images are displaying correctly +- MultiReference links between models and categories are correct + +- [ ] **Step 5: Run the script again (idempotency check)** + +```bash +node scripts/fetch-models.js +``` + +Expected: `Created: 0, Updated: 0, Deleted: 0, Unchanged: N` — second run should be a no-op. + +- [ ] **Step 6: Commit any final fixes** + +If any fixes were needed during integration testing, commit them: + +```bash +git add scripts/ +git commit -m "fix: address issues found during integration testing" +``` From e8d5ac3884af5fb23d0eff812d0da2fa8385460c Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:23:08 -0500 Subject: [PATCH 04/20] Update plan: fix API contracts and address review findings - Fix createItems payload: wrap in {items: [{fieldData}, ...]} - Fix deleteItems payload: use {itemIds: [...]} not {items: [{id}]} - Add chunk() utility for >100 item batching - Add defensive asset URL property check (hostedUrl/url/assetUrl) - Add missing test cases: name fallback, unknown modalities - Document cron runs target test site by default - Add transitional state note for Task 2 - Note about test:unit vs Playwright tests Co-Authored-By: Claude Opus 4.6 (1M context) --- .../plans/2026-03-23-webflow-cms-sync.md | 90 +++++++++++++++---- 1 file changed, 72 insertions(+), 18 deletions(-) diff --git a/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md b/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md index 107bad6..0d40428 100644 --- a/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md +++ b/docs/superpowers/plans/2026-03-23-webflow-cms-sync.md @@ -126,32 +126,51 @@ async function listCollectionItems(collectionId) { - [ ] **Step 4: Add batch create, update, delete, and publish** ```js +function chunk(array, size) { + const chunks = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; +} + async function createItems(collectionId, fieldDataArray) { - return webflowFetch(`/collections/${collectionId}/items`, { - method: 'POST', - body: JSON.stringify({ fieldData: fieldDataArray }), - }); + const results = []; + for (const batch of chunk(fieldDataArray, 100)) { + const data = await webflowFetch(`/collections/${collectionId}/items`, { + method: 'POST', + body: JSON.stringify({ items: batch.map((fieldData) => ({ fieldData })) }), + }); + results.push(...data.items); + } + return { items: results }; } async function updateItems(collectionId, itemsArray) { - return webflowFetch(`/collections/${collectionId}/items`, { - method: 'PATCH', - body: JSON.stringify({ items: itemsArray }), - }); + for (const batch of chunk(itemsArray, 100)) { + await webflowFetch(`/collections/${collectionId}/items`, { + method: 'PATCH', + body: JSON.stringify({ items: batch }), + }); + } } async function deleteItems(collectionId, itemIds) { - return webflowFetch(`/collections/${collectionId}/items`, { - method: 'DELETE', - body: JSON.stringify({ items: itemIds.map((id) => ({ id })) }), - }); + for (const batch of chunk(itemIds, 100)) { + await webflowFetch(`/collections/${collectionId}/items`, { + method: 'DELETE', + body: JSON.stringify({ itemIds: batch }), + }); + } } async function publishItems(collectionId, itemIds) { - return webflowFetch(`/collections/${collectionId}/items/publish`, { - method: 'POST', - body: JSON.stringify({ itemIds }), - }); + for (const batch of chunk(itemIds, 100)) { + await webflowFetch(`/collections/${collectionId}/items/publish`, { + method: 'POST', + body: JSON.stringify({ itemIds: batch }), + }); + } } ``` @@ -183,13 +202,16 @@ async function uploadAsset(siteId, fileName, fileBuffer) { throw new Error(`Asset upload to S3 failed (${uploadRes.status})`); } - return metadata.assetUrl; + // The response shape may vary — check for url, hostedUrl, or assetUrl + return metadata.hostedUrl || metadata.url || metadata.assetUrl; } ``` +> **Note:** The exact property name for the hosted URL in the Webflow Assets API response should be verified during integration testing (Task 6). The code defensively checks `hostedUrl`, `url`, and `assetUrl`. + - [ ] **Step 6: Export all functions and commit** -Ensure `createClient` returns all functions: `getCollections`, `findCollectionBySlug`, `listCollectionItems`, `createItems`, `updateItems`, `deleteItems`, `publishItems`, `uploadAsset`. +Ensure `createClient` returns all functions: `getCollections`, `findCollectionBySlug`, `listCollectionItems`, `createItems`, `updateItems`, `deleteItems`, `publishItems`, `uploadAsset`, `chunk`. ```bash git add scripts/webflow-api.js @@ -202,6 +224,10 @@ git commit -m "feat: add Webflow Data API client module" Build the pure functions for transforming API data to Webflow field format and diffing against existing items. +> **Note:** After this task, `scripts/fetch-models.js` will be in a transitional state — containing both the old pipeline code and the new exported pure functions. Task 3 does a full rewrite that replaces everything. + +> **Note:** Run unit tests with `node --test tests/fetch-models/`. The existing `pnpm test` runs Playwright e2e tests and is unrelated. + **Files:** - Create: `tests/fetch-models/transform.test.js` - Create: `tests/fetch-models/diff.test.js` @@ -271,6 +297,33 @@ describe('toWebflowFields', () => { assert.equal(result['model-url'], ''); assert.equal(result.logo, null); }); + + it('falls back to name when display_name is falsy', () => { + const model = { + name: 'test-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result.name, 'test-model'); + }); + + it('drops unknown modalities not in categoryMap', () => { + const model = { + display_name: 'Test', + name: 'test', + isLive: false, + isNew: false, + isTrending: false, + }; + const categoryMap = { llm: 'cat-1' }; + const modalities = ['LLM', 'UnknownModality']; + const result = toWebflowFields(model, modalities, categoryMap, null); + + assert.deepEqual(result.categories, ['cat-1']); + }); }); ``` @@ -812,6 +865,7 @@ Key changes: - Added `workflow_dispatch` inputs with environment choice - Added conditional env var selection for test vs production - Renamed job from `fetch` to `sync` +- **Scheduled runs** (cron): `inputs.environment` is undefined, so the conditional falls through to test secrets — scheduled runs always target the test site - [ ] **Step 2: Commit** From 1b3e1e7470f8de22f30b15c33f35f81171c8b6f7 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:33:07 -0500 Subject: [PATCH 05/20] feat: add Webflow Data API client module Co-Authored-By: Claude Sonnet 4.6 --- scripts/webflow-api.js | 189 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 scripts/webflow-api.js diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js new file mode 100644 index 0000000..e97aa3c --- /dev/null +++ b/scripts/webflow-api.js @@ -0,0 +1,189 @@ +import { createHash } from 'node:crypto'; + +function chunk(array, size) { + const chunks = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; +} + +function createClient(apiToken) { + async function webflowFetch(path, options = {}) { + const url = `https://api.webflow.com/v2${path}`; + const method = options.method || 'GET'; + + const response = await fetch(url, { + ...options, + method, + headers: { + Authorization: `Bearer ${apiToken}`, + 'Content-Type': 'application/json', + ...(options.headers || {}), + }, + }); + + if (!response.ok) { + const body = await response.text(); + throw new Error( + `Webflow API error: ${method} ${path} => ${response.status} ${response.statusText}\n${body}` + ); + } + + if (response.status === 204) { + return null; + } + + return response.json(); + } + + async function getCollections(siteId) { + const data = await webflowFetch(`/sites/${siteId}/collections`); + return data.collections; + } + + function findCollectionBySlug(collections, slug) { + const found = collections.find((c) => c.slug === slug); + if (!found) { + throw new Error(`Collection with slug "${slug}" not found`); + } + return found; + } + + async function listCollectionItems(collectionId) { + const limit = 100; + let offset = 0; + let allItems = []; + + while (true) { + const data = await webflowFetch( + `/collections/${collectionId}/items?limit=${limit}&offset=${offset}` + ); + const items = data.items || []; + allItems = allItems.concat(items); + + const total = data.pagination?.total ?? allItems.length; + if (allItems.length >= total || items.length === 0) { + break; + } + offset += limit; + } + + return allItems; + } + + async function createItems(collectionId, fieldDataArray) { + const batches = chunk(fieldDataArray, 100); + const allCreated = []; + + for (const batch of batches) { + const body = { + items: batch.map((fieldData) => ({ fieldData })), + }; + const data = await webflowFetch(`/collections/${collectionId}/items`, { + method: 'POST', + body: JSON.stringify(body), + }); + const created = data?.items || []; + allCreated.push(...created); + } + + return { items: allCreated }; + } + + async function updateItems(collectionId, itemsArray) { + const batches = chunk(itemsArray, 100); + const allUpdated = []; + + for (const batch of batches) { + const data = await webflowFetch(`/collections/${collectionId}/items`, { + method: 'PATCH', + body: JSON.stringify({ items: batch }), + }); + const updated = data?.items || []; + allUpdated.push(...updated); + } + + return { items: allUpdated }; + } + + async function deleteItems(collectionId, itemIds) { + const batches = chunk(itemIds, 100); + + for (const batch of batches) { + await webflowFetch(`/collections/${collectionId}/items`, { + method: 'DELETE', + body: JSON.stringify({ itemIds: batch }), + }); + } + } + + async function publishItems(collectionId, itemIds) { + const batches = chunk(itemIds, 100); + const allPublished = []; + + for (const batch of batches) { + const data = await webflowFetch( + `/collections/${collectionId}/items/publish`, + { + method: 'POST', + body: JSON.stringify({ itemIds: batch }), + } + ); + if (data?.publishedItemIds) { + allPublished.push(...data.publishedItemIds); + } + } + + return { publishedItemIds: allPublished }; + } + + async function uploadAsset(siteId, fileName, fileBuffer) { + // Step 1: Compute MD5 hash + const fileHash = createHash('md5').update(fileBuffer).digest('hex'); + + // Step 2: Request presigned upload URL + const metadata = await webflowFetch(`/sites/${siteId}/assets`, { + method: 'POST', + body: JSON.stringify({ fileName, fileHash }), + }); + + const { uploadUrl, uploadDetails } = metadata; + + // Step 3: Upload file via multipart/form-data to the presigned URL + const formData = new FormData(); + for (const [key, value] of Object.entries(uploadDetails)) { + formData.append(key, value); + } + formData.append('file', new Blob([fileBuffer]), fileName); + + const uploadResponse = await fetch(uploadUrl, { + method: 'POST', + body: formData, + }); + + if (!uploadResponse.ok) { + const body = await uploadResponse.text(); + throw new Error( + `Asset upload failed: ${uploadResponse.status} ${uploadResponse.statusText}\n${body}` + ); + } + + // Step 4: Return hosted URL + return metadata.hostedUrl || metadata.url || metadata.assetUrl; + } + + return { + webflowFetch, + getCollections, + findCollectionBySlug, + listCollectionItems, + createItems, + updateItems, + deleteItems, + publishItems, + uploadAsset, + }; +} + +export { createClient, chunk }; From 4fefe914727b090210580b41c26cde53c730136b Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:41:16 -0500 Subject: [PATCH 06/20] feat: add toWebflowFields and diffModels with tests --- scripts/fetch-models.js | 97 ++++++++++++++++++++++++---- tests/fetch-models/diff.test.js | 85 ++++++++++++++++++++++++ tests/fetch-models/transform.test.js | 96 +++++++++++++++++++++++++++ 3 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 tests/fetch-models/diff.test.js create mode 100644 tests/fetch-models/transform.test.js diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index fe028f3..bdcf957 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -2,11 +2,14 @@ import { writeFileSync, mkdirSync, rmSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; -const __dirname = dirname(fileURLToPath(import.meta.url)); +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const isMain = process.argv[1] === __filename; const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; -if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { +if (isMain && (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL)) { console.error('Missing required environment variables: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL'); process.exit(1); } @@ -107,15 +110,81 @@ async function processModelGarden(modelGarden) { return results; } -fetchModelGarden() - .then((data) => processModelGarden(data)) - .then((models) => { - const outDir = join(__dirname, '..', 'data'); - mkdirSync(outDir, { recursive: true }); - writeFileSync(join(outDir, 'models.json'), JSON.stringify(models, null, 2)); - console.log(`Wrote ${models.length} models to ${outDir}/models.json`); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); +if (isMain) { + fetchModelGarden() + .then((data) => processModelGarden(data)) + .then((models) => { + const outDir = join(__dirname, '..', 'data'); + mkdirSync(outDir, { recursive: true }); + writeFileSync(join(outDir, 'models.json'), JSON.stringify(models, null, 2)); + console.log(`Wrote ${models.length} models to ${outDir}/models.json`); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); +} + +export function toWebflowFields(model, modalities, categoryMap, logoField) { + return { + name: model.display_name || model.name, + slug: model.name, + 'display-name': model.display_name || '', + 'model-id': model.model_id || '', + 'logo-image': logoField, + description: model.description ? `

${model.description}

` : '', + provider: model.provider || '', + 'context-window': model.context_window || '', + 'total-params': model.total_params || '', + 'active-params': model.active_params || '', + precision: model.precision || '', + 'model-url': model.model_url || '', + live: model.isLive, + new: model.isNew, + trending: model.isTrending, + categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + }; +} + +const SKIP_DIFF_FIELDS = new Set(['logo-image', 'slug']); + +export function diffModels(apiModels, webflowItems) { + const wfBySlug = new Map(); + for (const item of webflowItems) { + wfBySlug.set(item.fieldData.slug, item); + } + + const toCreate = []; + const toUpdate = []; + let unchanged = 0; + const apiSlugs = new Set(); + + for (const model of apiModels) { + apiSlugs.add(model.slug); + const existing = wfBySlug.get(model.slug); + + if (!existing) { + toCreate.push(model.fields); + continue; + } + + const hasChanges = Object.keys(model.fields).some((key) => { + if (SKIP_DIFF_FIELDS.has(key)) return false; + const apiVal = model.fields[key]; + const wfVal = existing.fieldData[key]; + return JSON.stringify(apiVal) !== JSON.stringify(wfVal); + }); + + if (hasChanges) { + toUpdate.push({ id: existing.id, fieldData: model.fields }); + } else { + unchanged++; + } + } + + const toDelete = webflowItems + .filter((item) => !apiSlugs.has(item.fieldData.slug)) + .map((item) => item.id); + + return { toCreate, toUpdate, toDelete, unchanged }; +} diff --git a/tests/fetch-models/diff.test.js b/tests/fetch-models/diff.test.js new file mode 100644 index 0000000..c931d99 --- /dev/null +++ b/tests/fetch-models/diff.test.js @@ -0,0 +1,85 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { diffModels } from '../../scripts/fetch-models.js'; + +function makeApiModel(slug, fields) { + return { slug, fields: { slug, ...fields } }; +} + +function makeWfItem(id, slug, fieldData) { + return { id, fieldData: { slug, ...fieldData } }; +} + +describe('diffModels', () => { + it('identifies items to create when new in API but not in Webflow', () => { + const apiModels = [makeApiModel('new-model', { name: 'New Model' })]; + const webflowItems = []; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toCreate.length, 1); + assert.deepEqual(result.toCreate[0], { slug: 'new-model', name: 'New Model' }); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.toDelete.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('identifies items to delete when in Webflow but not in API', () => { + const apiModels = []; + const webflowItems = [makeWfItem('wf-id-1', 'old-model', { name: 'Old Model' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toDelete.length, 1); + assert.equal(result.toDelete[0], 'wf-id-1'); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('identifies items to update when fields differ', () => { + const apiModels = [makeApiModel('existing-model', { name: 'Updated Name' })]; + const webflowItems = [makeWfItem('wf-id-2', 'existing-model', { name: 'Old Name' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.toUpdate.length, 1); + assert.equal(result.toUpdate[0].id, 'wf-id-2'); + assert.deepEqual(result.toUpdate[0].fieldData, { slug: 'existing-model', name: 'Updated Name' }); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toDelete.length, 0); + assert.equal(result.unchanged, 0); + }); + + it('counts unchanged items when fields are identical', () => { + const apiModels = [makeApiModel('same-model', { name: 'Same Name', provider: 'Acme' })]; + const webflowItems = [makeWfItem('wf-id-3', 'same-model', { name: 'Same Name', provider: 'Acme' })]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toCreate.length, 0); + assert.equal(result.toUpdate.length, 0); + assert.equal(result.toDelete.length, 0); + }); + + it('ignores logo-image field differences in comparison', () => { + const apiModels = [ + makeApiModel('logo-model', { + name: 'Logo Model', + 'logo-image': { fileId: 'new-file-id', url: 'https://cdn.example.com/new.png' }, + }), + ]; + const webflowItems = [ + makeWfItem('wf-id-4', 'logo-model', { + name: 'Logo Model', + 'logo-image': { fileId: 'old-file-id', url: 'https://cdn.example.com/old.png' }, + }), + ]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toUpdate.length, 0); + }); +}); diff --git a/tests/fetch-models/transform.test.js b/tests/fetch-models/transform.test.js new file mode 100644 index 0000000..6540783 --- /dev/null +++ b/tests/fetch-models/transform.test.js @@ -0,0 +1,96 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { toWebflowFields } from '../../scripts/fetch-models.js'; + +describe('toWebflowFields', () => { + it('maps all fields correctly for a full model', () => { + const model = { + display_name: 'My Model', + name: 'my-model', + model_id: 'org/my-model', + description: 'A great model', + provider: 'Acme', + context_window: 8192, + total_params: '70B', + active_params: '7B', + precision: 'fp16', + model_url: 'https://example.com/model', + isLive: true, + isNew: false, + isTrending: true, + }; + const modalities = ['Text', 'Image']; + const categoryMap = { text: 'id-text', image: 'id-image' }; + const logoField = { fileId: 'abc', url: 'https://cdn.example.com/logo.png' }; + + const result = toWebflowFields(model, modalities, categoryMap, logoField); + + assert.equal(result.name, 'My Model'); + assert.equal(result.slug, 'my-model'); + assert.equal(result['display-name'], 'My Model'); + assert.equal(result['model-id'], 'org/my-model'); + assert.deepEqual(result['logo-image'], logoField); + assert.equal(result.description, '

A great model

'); + assert.equal(result.provider, 'Acme'); + assert.equal(result['context-window'], 8192); + assert.equal(result['total-params'], '70B'); + assert.equal(result['active-params'], '7B'); + assert.equal(result.precision, 'fp16'); + assert.equal(result['model-url'], 'https://example.com/model'); + assert.equal(result.live, true); + assert.equal(result.new, false); + assert.equal(result.trending, true); + assert.deepEqual(result.categories, ['id-text', 'id-image']); + }); + + it('handles undefined optional fields with empty string defaults', () => { + const model = { + display_name: 'Minimal Model', + name: 'minimal-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result['model-id'], ''); + assert.equal(result.description, ''); + assert.equal(result.provider, ''); + assert.equal(result['context-window'], ''); + assert.equal(result['total-params'], ''); + assert.equal(result['active-params'], ''); + assert.equal(result.precision, ''); + assert.equal(result['model-url'], ''); + assert.deepEqual(result.categories, []); + }); + + it('falls back to name when display_name is falsy', () => { + const model = { + display_name: '', + name: 'fallback-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const result = toWebflowFields(model, [], {}, null); + + assert.equal(result.name, 'fallback-model'); + assert.equal(result['display-name'], ''); + }); + + it('drops unknown modalities not in categoryMap', () => { + const model = { + display_name: 'Multi Model', + name: 'multi-model', + isLive: false, + isNew: false, + isTrending: false, + }; + const modalities = ['Text', 'Video', 'Audio']; + const categoryMap = { text: 'id-text', audio: 'id-audio' }; + + const result = toWebflowFields(model, modalities, categoryMap, null); + + assert.deepEqual(result.categories, ['id-text', 'id-audio']); + }); +}); From be5561b4c2ccce331eac699129b43bb515da3dfa Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:55:48 -0500 Subject: [PATCH 07/20] feat: rewrite fetch-models.js for Webflow CMS sync --- scripts/fetch-models.js | 257 +++++++++++++++++++++++++++------------- 1 file changed, 176 insertions(+), 81 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index bdcf957..ffc6fcd 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -1,67 +1,53 @@ -import { writeFileSync, mkdirSync, rmSync } from 'fs'; -import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; +import { createClient } from './webflow-api.js'; const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - const isMain = process.argv[1] === __filename; +// -- Environment variables -- + const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; +const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID } = process.env; -if (isMain && (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL)) { - console.error('Missing required environment variables: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL'); - process.exit(1); +let wf; +if (isMain) { + if (!MODULAR_CLOUD_API_TOKEN || !MODULAR_CLOUD_ORG || !MODULAR_CLOUD_BASE_URL) { + console.error( + 'Missing required env vars: MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL' + ); + process.exit(1); + } + if (!WEBFLOW_API_TOKEN || !WEBFLOW_SITE_ID) { + console.error('Missing required env vars: WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID'); + process.exit(1); + } + wf = createClient(WEBFLOW_API_TOKEN); } -const JSDELIVR_BASE = 'https://cdn.jsdelivr.net/gh/modularml/modular-webflow@master/data/images'; - -const headers = { +const modularHeaders = { 'X-Yatai-Api-Token': MODULAR_CLOUD_API_TOKEN, 'X-Yatai-Organization': MODULAR_CLOUD_ORG, }; +// -- Modular Cloud API -- + async function fetchModelGarden() { - const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { headers }); + const countRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden`, { + headers: modularHeaders, + }); if (!countRes.ok) throw new Error(`Count request failed: ${countRes.status}`); const { total } = await countRes.json(); const listRes = await fetch(`${MODULAR_CLOUD_BASE_URL}/api/v1/model_garden?count=${total}`, { - headers, + headers: modularHeaders, }); if (!listRes.ok) throw new Error(`List request failed: ${listRes.status}`); return listRes.json(); } -const MIME_TO_EXT = { - 'image/png': '.png', - 'image/svg+xml': '.svg', - 'image/jpeg': '.jpg', - 'image/jpg': '.jpg', - 'image/gif': '.gif', - 'image/webp': '.webp', -}; - -function parseDataUri(dataUri) { - if (!dataUri || !dataUri.startsWith('data:')) return null; - - const match = dataUri.match(/^data:([^;]+);base64,(.+)$/s); - if (!match) return null; - - const [, mime, payload] = match; - const ext = MIME_TO_EXT[mime]; - if (!ext) return null; - - const buffer = Buffer.from(payload, 'base64'); - if (buffer.length === 0) return null; - - return { mime, ext, buffer }; -} - function transformModel(model) { const meta = model.metadata || {}; const tags = meta.tags || []; - return { display_name: model.display_name, name: model.name, @@ -75,55 +61,13 @@ function transformModel(model) { active_params: meta.active_params, precision: meta.precision, model_url: meta.model_url, - pricing: model.pricing, isLive: Boolean(model.gateway_id), isNew: tags.includes('New'), isTrending: tags.includes('Trending'), }; } -async function processModelGarden(modelGarden) { - const results = []; - const imagesDir = join(__dirname, '..', 'data', 'images'); - rmSync(imagesDir, { recursive: true, force: true }); - mkdirSync(imagesDir, { recursive: true }); - - for (const model of modelGarden.items) { - const transformed = transformModel(model); - - const parsed = parseDataUri(transformed.logo_url); - if (parsed) { - try { - const filename = `${transformed.name}${parsed.ext}`; - writeFileSync(join(imagesDir, filename), parsed.buffer); - transformed.logo_url = `${JSDELIVR_BASE}/${filename}`; - console.log(`Saved image for ${transformed.name}: ${filename}`); - } catch (err) { - console.error(`Failed to save image for ${transformed.name}: ${err.message}`); - transformed.logo_url = null; - } - } - - results.push(transformed); - } - - return results; -} - -if (isMain) { - fetchModelGarden() - .then((data) => processModelGarden(data)) - .then((models) => { - const outDir = join(__dirname, '..', 'data'); - mkdirSync(outDir, { recursive: true }); - writeFileSync(join(outDir, 'models.json'), JSON.stringify(models, null, 2)); - console.log(`Wrote ${models.length} models to ${outDir}/models.json`); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); -} +// -- Field mapping -- export function toWebflowFields(model, modalities, categoryMap, logoField) { return { @@ -146,6 +90,8 @@ export function toWebflowFields(model, modalities, categoryMap, logoField) { }; } +// -- Diff -- + const SKIP_DIFF_FIELDS = new Set(['logo-image', 'slug']); export function diffModels(apiModels, webflowItems) { @@ -188,3 +134,152 @@ export function diffModels(apiModels, webflowItems) { return { toCreate, toUpdate, toDelete, unchanged }; } + +// -- Logo resolution -- + +const MIME_TO_EXT = { + 'image/png': '.png', + 'image/svg+xml': '.svg', + 'image/jpeg': '.jpg', + 'image/jpg': '.jpg', + 'image/gif': '.gif', + 'image/webp': '.webp', +}; + +async function resolveLogo(model) { + const { logo_url, display_name, name } = model; + + if (!logo_url) return null; + + if (logo_url.startsWith('http')) { + return { url: logo_url, alt: `${display_name || name} logo` }; + } + + if (logo_url.startsWith('data:')) { + const match = logo_url.match(/^data:([^;]+);base64,(.+)$/s); + if (!match) return null; + + const [, mime, payload] = match; + const ext = MIME_TO_EXT[mime]; + if (!ext) return null; + + const buffer = Buffer.from(payload, 'base64'); + if (buffer.length === 0) return null; + + console.log(`Uploading logo for: ${name}`); + const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${name}${ext}`, buffer); + return { url: assetUrl, alt: `${display_name || name} logo` }; + } + + return null; +} + +// -- Categories sync -- + +async function syncCategories(models, categoriesCollectionId) { + const allModalities = new Set(); + for (const model of models) { + if (model.modalities) { + for (const m of model.modalities) allModalities.add(m); + } + } + + const existingItems = await wf.listCollectionItems(categoriesCollectionId); + const existingBySlug = new Map(); + for (const item of existingItems) { + existingBySlug.set(item.fieldData.slug, item.id); + } + + const categoryMap = {}; + for (const modality of allModalities) { + const slug = modality.toLowerCase(); + if (existingBySlug.has(slug)) { + categoryMap[slug] = existingBySlug.get(slug); + } else { + console.log(`Creating category: ${modality}`); + const result = await wf.createItems(categoriesCollectionId, [{ name: modality, slug }]); + const newItem = result.items[0]; + categoryMap[slug] = newItem.id; + await wf.publishItems(categoriesCollectionId, [newItem.id]); + } + } + + return categoryMap; +} + +// -- Main -- + +async function main() { + // Phase 1: Fetch + console.log('Fetching models from Modular Cloud API...'); + const modelGarden = await fetchModelGarden(); + const models = modelGarden.items.map(transformModel); + console.log(`Fetched ${models.length} models`); + + // Discover collections + const collections = await wf.getCollections(WEBFLOW_SITE_ID); + const categoriesCol = wf.findCollectionBySlug(collections, 'models-categories'); + const modelsCol = wf.findCollectionBySlug(collections, 'models'); + + // Sync categories + console.log('Syncing categories...'); + const categoryMap = await syncCategories(models, categoriesCol.id); + console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); + + // Resolve logos and build field data + console.log('Resolving logos and building field data...'); + const apiModels = []; + for (const model of models) { + const logoField = await resolveLogo(model); + const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); + apiModels.push({ slug: model.name, fields }); + } + + // Phase 2: Diff + console.log('Fetching existing Webflow items...'); + const webflowItems = await wf.listCollectionItems(modelsCol.id); + const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); + + // Phase 3: Sync + const publishIds = []; + + if (toCreate.length > 0) { + console.log(`Creating ${toCreate.length} models...`); + for (const fields of toCreate) { + console.log(` Creating: ${fields.slug}`); + } + const created = await wf.createItems(modelsCol.id, toCreate); + publishIds.push(...created.items.map((item) => item.id)); + } + + if (toUpdate.length > 0) { + console.log(`Updating ${toUpdate.length} models...`); + for (const item of toUpdate) { + console.log(` Updating: ${item.fieldData.slug}`); + } + await wf.updateItems(modelsCol.id, toUpdate); + publishIds.push(...toUpdate.map((item) => item.id)); + } + + if (toDelete.length > 0) { + console.log(`Deleting ${toDelete.length} models...`); + await wf.deleteItems(modelsCol.id, toDelete); + } + + if (publishIds.length > 0) { + console.log(`Publishing ${publishIds.length} items...`); + await wf.publishItems(modelsCol.id, publishIds); + } + + // Summary + console.log( + `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` + ); +} + +if (isMain) { + main().catch((err) => { + console.error(err); + process.exit(1); + }); +} From 5e4507f9cf96f1cabd99d9870e1725023d667393 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 09:59:17 -0500 Subject: [PATCH 08/20] feat: update workflow for Webflow CMS sync - Remove git commit/push step (no more models.json writes) - Remove contents:write permission (not needed) - Add WEBFLOW_API_TOKEN and WEBFLOW_SITE_ID env vars - Use TEST_WEBFLOW_API_TOKEN secret and TEST_WEBFLOW_SITE_ID var - Rename job from fetch to sync --- .github/workflows/fetch-models.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/fetch-models.yml b/.github/workflows/fetch-models.yml index 995e514..190428c 100644 --- a/.github/workflows/fetch-models.yml +++ b/.github/workflows/fetch-models.yml @@ -6,10 +6,8 @@ on: workflow_dispatch: jobs: - fetch: + sync: runs-on: ubuntu-latest - permissions: - contents: write steps: - uses: actions/checkout@v4 @@ -17,17 +15,11 @@ jobs: with: node-version: '20' - - name: Fetch models + - name: Sync models to Webflow run: node scripts/fetch-models.js env: MODULAR_CLOUD_API_TOKEN: ${{ secrets.MODULAR_CLOUD_API_TOKEN }} MODULAR_CLOUD_ORG: ${{ vars.MODULAR_CLOUD_ORG }} MODULAR_CLOUD_BASE_URL: ${{ vars.MODULAR_CLOUD_BASE_URL }} - - - name: Commit and push - run: | - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git add data/models.json data/images/ - git diff --cached --quiet || git commit -m "Update models.json" - git push + WEBFLOW_API_TOKEN: ${{ secrets.TEST_WEBFLOW_API_TOKEN }} + WEBFLOW_SITE_ID: ${{ vars.TEST_WEBFLOW_SITE_ID }} From 3c1aee3fa9a861179991c8e5ea49df35ab86a967 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:00:23 -0500 Subject: [PATCH 09/20] chore: remove models.json and data/images (replaced by Webflow CMS) --- data/images/flux2.svg | 1 - data/images/smollm-135m-instruct-fp32.png | Bin 8175 -> 0 bytes data/models.json | 851 ---------------------- 3 files changed, 852 deletions(-) delete mode 100644 data/images/flux2.svg delete mode 100644 data/images/smollm-135m-instruct-fp32.png delete mode 100644 data/models.json diff --git a/data/images/flux2.svg b/data/images/flux2.svg deleted file mode 100644 index e42a6b2..0000000 --- a/data/images/flux2.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/data/images/smollm-135m-instruct-fp32.png b/data/images/smollm-135m-instruct-fp32.png deleted file mode 100644 index 3adb37681e07f2aa08814fc56332943ca945385a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8175 zcmdUUWmHsO*zW;{ZbiCVq{E>ckVZ;6C4`|%8XaIPkdp3hM7kRk7#gIcI|m7AhMIT& zU+!A>!~6Z-b=E#RpIzt4-wAuAr9whTM+g7_iJGdSE++57q)vQX%)6vy@)0Iuu~F64 z1b}y30093C0N0o%_$~nW2m!#J6#z)3006Z|MvIONrUUnlx{4xz{_n|YFOJ8waC$2l zd)r!jL#1pyp_mL15fXXEFC@Y*BBn1SEF~r)B?NvdBqSvyR5Xz1_kR(%y4yP11^n*> zGYND<7y_REvx2vyE7Z%|+STKKr-6TvrUU?{bv4Bo`u_6=nF0O`)0q`3rOU1LLvbzx zNv{0Mu8}5@t`}G~Ral=s$c0qdHGLoecax~Oae?F2esOe*n54?cU?Iryevx+}G^*wv z`qt8H1jh24CAB%!Ul_U@_@GNl8*p^}2I2}0X9BW|`AwD#0!P1J@40jyF(Bvcv!}`N zUBW^5SokgxUVj{mqrK@sA1M^%Nx#y`S(#;cc7Z1x$?=tmi+8c{_akK{2r!k0tiXqI z3T*OOy8qV-JF46)v~e7nX|BT@zs1Pk7pSQk`bB7Ui>NzdInb5N|E6yE9KiY_Nm(kH zJ)+bx$7E4>=_(^Z8Jbjs{mOu(azRO}%h^07m+B)(Rjl4G*U{AUwf<9wG>cZME!&)A z*SwQyB9%B6hOqJZ=}#G4+N5fu!`jW)8V2GpChF&XbXq7s=;O$Ib{eg8BB#d&)BU9W zj_xudt2TEi8-frvM;*H-Mh{9#5ly!Y-~Jr#w65HAE_#hEDYEHsApON%Wun*n92`_( zs5cDgGT&clPJPYZjpHilkYPoOC+!AzqE}O4)OfIDd9I^Su26K#gB#uD$~W{U>x&&pD(+=RR(7nGoP0>G zQ+Y{=6)OC%q^C#l?Lf+??NvJiq$%6k(0nZ6Nn%p;)ysE4)8V}HzZ+OdsS^sB{cRS!!z~niKL;qX>!?mj zVyR(MMj7+rcBR@QN3zEkKQI!y@B_7$!j9{TZmk7jTYY5YWb+=Yq%?eJ>kVq!=s`?;B7v|^LHx6`9?IwL zUE)KPh%L|D($cM;U#@l{nZxtsI6glN&kJF@>ztaZc%$n{xofLqFuhyGm@VSPK|2E$ z+;0mge(>u3CeK`eNu3|g&TC9y?sJ;J>!JS zK~>I|ZfWUe9<57?`4l3ewJDw_-z?74^i;-MGWkm^4|7Z~U8n zO*|FoTk|lcyBimqZ3v`iWMq`cye?n&eeZu8#!Z0x>3Y)gz(!t!OsE&>UE_B5&LSW{ z+ytlB`{J{n8=SQq^lx)e_8&9ZUE9pc9z7ioFP|72cZ&hd&jo zs`iffG5r_s=NjL0g9uYG`3DgYRkc7_m9WfFBYxGn(Kj>UT^)KC{43iL^4FsSS3JnNz0FOU2& zpWR0=+3Q2_6JFlE#GMYIA*UZYzC_fSeQl!Wx$9Y3ENMlYfvKVgrFrNn-Ksk>M=roC zvbE#;m%TJ8;7@rlqb9qxkDyu{q!i2i9kLK;GS~Xrqk9-8PR^nB(SbO59M*gm*ZJU# zB0~`s2c;!3ZoC%wJ!<2Z61=_Dm3 z7XubXfJnh%6(SYYr5z(SjyjLeae2bPqEDae=Jqz%0Z}UW3NZ$Gz``v}lvhpZ+C`jI zM+wJW^?bBHuHc70*^|l}r>WUjy#k4@v|i;!M4^AEGJ(HK1i2Pca#Ye!JIufKUi;F1 zGHZFyNQyV^kYI$aA%$DwL^W0~WdJ_(Oiz^YtSrm!Y%NWc=& z@w_$VRSNS_e-xL#5>);LWZIy$X|v{ONmhaJ^4Ca$PI| z2NRxnBKHYIrJUuQ3+e$egD86Qtl%}yk3X$$xsEPrHrl@sQ3D=j<(b5t_64;fRw8#6 zrlt#kgx^}8MZjOdl{k*^M}*beTlK(aHi|m2RO0>UIx6^njO5g;yvdf>pNgOldv`Y< zY}skA^{VpPGAb9nMWaH=zP-%X=-SCes`7&u&kVSdH4hUTF$XIsbrKYmK{G%TCrex% zvk{b;#Pb3QYdn0wC8UY}#W6d}z1c6V`q*MQvxY zzgQnZa@47SnnHTl`OwDZAj{!m)bQ~JAL%fDTu15{-Ud{Sa=Y&Eq;O zH&FEj0Omij4i=AE9eo>dS!h@~dCSSkd3#D0guh0L*xZ)m?Zf>}XX9DRIx%{QMxjZ% zuJTlZ{^%bR!1jpeKjp@b4pdpwV<~oz`x~GeJKM|cEs}1^x3$Yy1zL8o0eyp?TUxJ) zz6@U4Vqo;b@A_3I<{Vf$ZFKm@j&5J@sY$q%_epD%PEQf^^g zrkguOn(07_@OwJ`sPYC=o*{^&(t~1+P-`!9^gKO}8mziSN4axOPy!7DGPG0%X`viy z@)3$}rO&yRaj1744JgIGA1h7QH60h`|Dex6%AU>rAXb*b>P|CBj!-xsp-j}+9ACK& zpZ`=3ub2FgDrzs{e$Q6=!#6fIGgdbwKuKp{aKzSfT42<~8&YdOvX^+#A#`f4r`H{P ze*vaUO7JYXr>S`wsE~d0%Mcroo5Tm*0|LN~NcG$m> zdjvYhx`U;~Mm6r1;#MO(dXj~4Ka9-B?n_^5@C>S`e;BPQElK#s*o`rI&7LU2BCX7c zrz3&Z+x0E)WP@SXPYA=+k$cr+v*@r{T_E!(d4QxO06VK*235UDIKF|IJD0KUoj0*Q zR#vwq$X2<0TGn*ogdNlg-5A27hk#gQ1w*yyHG?u$3C({}$W5+~^Kqm zPi;BNgd}VulJBeRD`J1kQ`nl2ISa8X#*5SI8}uId*18}c!w;zq*@czu#8l;@de%Eu zpz4)f2fo~|PZnWEmC6!dyOD|q1eeVtWsFtdfwS0ng{TogrhU(3Ub>7PIx3;yj_c!t z_k!uCnM%^GPy_$-zr&*I0e`_W%BRKU?U@E+GI(8U2r7r&Z&S{ao}<{C_=#7|i-R>4*?Zv)L{vR&H>unV zSyGw%W}aAzc%v<|x*f=iHEwf7U(>rIxSQca;!5@!!l1-uY+!S2-0U&aa*YKD)t)^VX$^ zOn2_XREQv^ChE9Xb``f{DB*d=gFgPJAc5U1*4VHxcs0S?Kn>>`s(aQb3OxY<5b-1& z?O8G*r?0{&)S}N#Ib4`qh?sSvH1DAh;{K@P)DRI#YZEYVs+oQUT`$%WD6B6W`I9fp zl?VFWwY6&eo*QtU>5%86QePxqzlvG^5`yY^GDQBoUYXXbkGlBnMw)sScls^H-!Gc+AInMx zPkh}PhdvU2TrnkcZFpG2tplkHEG2jj8tTWeP!=}50(UG14UO1BO~D3F3`U*Rwa&zjObBooET957*BGeo5Qo4YaJ* z`%OGSnRtv7Fr|OAmG;*yXV~~;ac~mdb^oC3^(mVTw?`>si&fVpNOuNNA84u&WMCH?cl|A_ zUlVFvIribwPMiAMdb8fcX!)$|@>$}fooC6h@%+)1$SXxhSgb1XbW^}G?5{gqJU z{S82qu~tw0Lp)SyCovqqH~1q7+1h9<*3ou1Dh(gsdYnsUDUZKOq{>7cN|%Pk-FdyV zQO6*ikl;#(A`KI@7E>V|B$Pt*LNCFdVUCjxiA9?5*t$Y@XJbsyv z!wMH<8G5+(pX6%gu30@^XgmVSG{ZG%WV6s-Xg^0+2@S__nyLQP9nh3&0)Y~$4$QRH z;l9Fbc_K5unVkCPK-%~ByOOk%CtIIe4{Ep3a^%=o*l9(q&96VRUtg3~w%<#|^zd{a z)6NW0SrW>H1b>YbNmK4Jj(s-(jfnrF%uQ#x?5_}g8>yJz{|wx57D_J1tXfkRovdyV zbni~)vDvSKQAbyGliPC3625LL^42=bIguizr8>qbeH900$J^SgXU$Z;FyRURtA2{0 z3wJU&dye2cPVALy#e|pH4w$%1Uu!y)Tmi9wrM*1r;ibqOpaWfw zR(kQ1W+>Ecsf7eSIH=~>xo~4ej=dXdEJoD{{nL3N6`n`zuac_D5#IOygi$+sMrmqS z#oRl$SzXg}BO;Hd44fo015cs3LsGNCgJKvSlJJ*rQYHRdutdkTD1-uh{9mP=w>3sh z{h9DC4wu+`*NA{_8xwXLsAFMB~aPi-W4Vrnwh>_a>0G=aUTE|HkXvkk933yyZfG+ z4=4xQnsCuYot(@Y0X7hM#UL(4d=@OymHq?!bsK=h8?eC;NvD&WPXoi!+aireMYowU}{83qEtv$V5kPXhBdc3s-J9lo> zZ4DOOAUffrg<>dQk3c$n7Y-5<6VIjW*)H@Aj;~ufg{(clV89z%9(jNY|3VS}>OAUP z!z+wK?GEurJ??XnlfMj8g=qt2m0LnAgNQ)>Qc!Okr~l&NX9V@Z=XUtWwszMM_v|BD z+0)~;DBH3XMQ^)=@_>m#Lniptj8AB&QR(OTGg-_IQ^{2 z-Hn?5^Ji4=hxdlMy&pIaX+y#a5cQL$vPbc2WD2Kaz`Op@rB*EG>KD*>?&GNUhlRB$ z9xTf@-EFuam_>^jUDFNo5o63Tv~QHr<#so)Xe47~?V8FOR{fQSQ(cu@?^R^E>PnCq zIMrBqP=bN9qG?fR(0|Ec+&H_nT2<8{mIh*T;ye)7i|HFeB>OD~lKq}}qNJl1^*mCK znH5lX=(hlipKGp;tl^v`Zx!1D|P$iFrslLjK@(wM}YkXYuB{ z|DKxaQRuv8?1wnfr@6Lo49QVoSAFGB>v{J}UxPQq!C7X0Vz6NL>X;LT#*-~5KgN%b zAkyi!Jj^Lwuz7RcqVFP^DO90SbNTwm;p;;J zm6$~VYZxsc4?K{IW1%7ZAQ&3TbSqh3?*p$N{WnZIX+13A>$>vN`h{C1AZako!`BbJ z{8lb;s-7b5Wmjo)^Xqn0!N*7Dc6ID$x`Gg;yv!|L2zeKP#1Bak36KG@-IrZZO&>pE zGhHv=!L1V{b;Rc1ZVYL6p7IkC5%OhUih2#4))3VBMGRDaU;_!sC6qiQb9wKx^e%5a z1)4vM+Clye2vM+Zv+%nWwWv(58W(KpGIm=EKjx&0==Fo?=;MeNMS34cN__@!zi!5S zcXuqTYD!p=&+ zI_&vY1f&^O6MARSYHT2nxPDT}j-lI#A2Yul`3kejP}o%txyw@n_@I8FUu9#d!M+nm z`Ua0p;*k4$>z$VV6qvwX71Tk3t>gJAN0cTHqYxV{EXNL?zmE%obR=B&=vK7fKXtj^ z*Mp$<^^z)^vcev5UtDXj`5rC~EVy@!_MAL<(|GxUjbZW=CPYLWJq4|1`YmsiiH=43 zh1|5nj@ZR49BePUFXy{^75v1&{p(tGFiC6M-6g;C?dX}mtd!wj>8!}{v4{!7E=C0a zCQFo<-(t{HOxzR}9m65F@#}PK2a{$PUn3-95{rYKou586dj5bMjKtSsI^LxL)hMhx zC=gBBjTSleQ8kztzkIDi{oy{{c8-s%D)5HX*LA6vHZ14V0QkDMt%96cjBXEu#dS_LLI~u@_>S(N>yk$K}+;y zyfy@8-*2yv=U!0ent-kKX2P%6DJm=97^t8YhwlR#80-hEx^b7vCM;XZ z+lIe14SPZDq2zn`-@YFA@|hY;51D9NqLI%+-A0^KN|(+PDB;Z&QA6c7b{IgEH1vT< zQsG7z|aJKLX%sEIq8J?pP`^lZdn#BwsmAdTI$ax#ozsD& z@ef(J%1bV69%xuNP)_1#-)9UgUTCwp7WY1HDf+HF*o%!+kKVOLOixd{%%4eOfLGgY zrZ|qbRog5%q4*mNL9RnJ8fGeBoZiA)j1g_Cvm?=aTKxMF;aUuIsIeTxQyM<@x6tXx zV)>mbBW93Ty)PTt`ws6Tg~svYisjC1ZOBqen|r=vles|Y4>F)&zdw<~e!JNGpn#V* z6^l7jQw101f=K!Ar-SmUvLx$WsdC7Sg3-`8tE8pj3t`43)=i4V1Dz;GTiiD9XJof& zwp5gpb4&4YZ`GeX0b+(=xmj<<^ zrJ1){$97#Me7y@R-@6<-xhy%ht^sZ~HI~-0^me@w>Zd~@6D;j}?~-)6E%WcvV6h2Q zKtT`|ms+6d;e

rJB2Lr4buBF}pl~ZN?5_sS^MHqi~AGYFTED#Lpn0#{3}%)ReRo JE99*{{tukfct`*M diff --git a/data/models.json b/data/models.json deleted file mode 100644 index 8af10d0..0000000 --- a/data/models.json +++ /dev/null @@ -1,851 +0,0 @@ -[ - { - "display_name": "FLUX.2 Dev", - "name": "flux2", - "model_id": "black-forest-labs/FLUX.2-dev", - "logo_url": "https://cdn.jsdelivr.net/gh/modularml/modular-webflow@master/data/images/flux2.svg", - "modalities": [ - "Image" - ], - "total_params": "32B", - "precision": "BF16", - "model_url": "https://huggingface.co/black-forest-labs/FLUX.2-dev", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": true, - "isNew": false, - "isTrending": true - }, - { - "display_name": "smollm", - "name": "smollm-135m-instruct-fp32", - "description": "smollm-135m-instruct-fp32", - "model_id": "modularai/smollm-135m-instruct-fp32", - "logo_url": "https://cdn.jsdelivr.net/gh/modularml/modular-webflow@master/data/images/smollm-135m-instruct-fp32.png", - "provider": "Modular", - "modalities": [ - "LLM" - ], - "context_window": "128k", - "precision": "FP8", - "pricing": { - "cached_input": "1", - "input": "1", - "output": "1" - }, - "isLive": true, - "isNew": false, - "isTrending": false - }, - { - "display_name": "GLM-5", - "name": "glm-5", - "description": "GLM-5 by Zhipu AI is a 744B MoE model with 44B active parameters, featuring strong reasoning capabilities across multilingual tasks.", - "model_id": "zai-org/GLM-5", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/62dc173789b4cf157d36ebee/i_pxzM2ZDo3Ub-BEgIkE9.png", - "provider": "Zhipu AI", - "modalities": [ - "LLM" - ], - "context_window": "200K", - "total_params": "744B", - "active_params": "44B", - "precision": "FP8", - "model_url": "https://huggingface.co/zai-org/GLM-5", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": true - }, - { - "display_name": "Kimi K2.5", - "name": "kimi-k2-5", - "description": "Kimi K2.5 by Moonshot AI is a ~1T MoE model with 32B active parameters, supporting text and vision with reasoning capabilities.", - "model_id": "moonshotai/Kimi-K2.5", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/641c1e77c3983aa9490f8121/X1yT2rsaIbR9cdYGEVu0X.jpeg", - "provider": "Moonshot AI", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "~1T", - "active_params": "32B", - "precision": "BF16 / INT4", - "model_url": "https://huggingface.co/moonshotai/Kimi-K2.5", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": true - }, - { - "display_name": "MiniMax M2.5", - "name": "minimax-m2-5", - "description": "MiniMax M2.5 is a 230B MoE model with 10B active parameters, optimized for efficient text generation.", - "model_id": "MiniMaxAI/MiniMax-M2.5", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/676e38ad04af5bec20bc9faf/dUd-LsZEX0H_d4qefO_g6.jpeg", - "provider": "MiniMax", - "modalities": [ - "LLM" - ], - "context_window": "200K", - "total_params": "230B", - "active_params": "10B", - "precision": "BF16", - "model_url": "https://huggingface.co/MiniMaxAI/MiniMax-M2.5", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": true - }, - { - "display_name": "DeepSeek V3.2", - "name": "deepseek-v3-2", - "description": "DeepSeek V3.2 is a 685B MoE model with 37B active parameters, excelling at code, math, and general reasoning tasks.", - "model_id": "deepseek-ai/DeepSeek-V3.2", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/6538815d1bdb3c40db94fbfa/xMBly9PUMphrFVMxLX4kq.png", - "provider": "DeepSeek", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "685B", - "active_params": "37B", - "precision": "FP8 / NVFP4", - "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": true - }, - { - "display_name": "Qwen3.5-397B-A17B", - "name": "qwen3-5-397b-a17b", - "description": "Qwen3.5-397B-A17B by Alibaba is a 397B MoE model with 17B active parameters, supporting text, vision, and video with hybrid reasoning.", - "model_id": "Qwen/Qwen3.5-397B-A17B", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "262K", - "total_params": "397B", - "active_params": "17B", - "precision": "FP8 / NVFP4", - "model_url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-235B-A22B", - "name": "qwen3-235b-a22b", - "description": "Qwen3-235B-A22B by Alibaba is a 235B MoE model with 22B active parameters, featuring hybrid reasoning for text tasks.", - "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM" - ], - "context_window": "262K", - "total_params": "235B", - "active_params": "22B", - "precision": "FP8", - "model_url": "https://huggingface.co/Qwen/Qwen3-235B-A22B-Instruct-2507", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "DeepSeek V3.1", - "name": "deepseek-v3-1", - "description": "DeepSeek V3.1 is a 671B MoE model with 37B active parameters, supporting hybrid reasoning for text generation.", - "model_id": "deepseek-ai/DeepSeek-V3.1", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/6538815d1bdb3c40db94fbfa/xMBly9PUMphrFVMxLX4kq.png", - "provider": "DeepSeek", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "671B", - "active_params": "37B", - "precision": "FP8", - "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "GLM-4.7", - "name": "glm-4-7", - "description": "GLM-4.7 by Zhipu AI is a 355B MoE model with 32B active parameters, supporting text, vision, and audio with reasoning.", - "model_id": "zai-org/GLM-4.7", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/62dc173789b4cf157d36ebee/i_pxzM2ZDo3Ub-BEgIkE9.png", - "provider": "Zhipu AI", - "modalities": [ - "LLM", - "Vision", - "Audio" - ], - "context_window": "200K", - "total_params": "355B", - "active_params": "32B", - "precision": "BF16 / INT4", - "model_url": "https://huggingface.co/zai-org/GLM-4.7", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "DeepSeek R1-0528", - "name": "deepseek-r1-0528", - "description": "DeepSeek R1-0528 is a 671B MoE reasoning model with 37B active parameters, specialized in chain-of-thought reasoning.", - "model_id": "deepseek-ai/DeepSeek-R1-0528", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/6538815d1bdb3c40db94fbfa/xMBly9PUMphrFVMxLX4kq.png", - "provider": "DeepSeek", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "671B", - "active_params": "37B", - "precision": "FP8", - "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Llama 4 Maverick", - "name": "llama-4-maverick", - "description": "Llama 4 Maverick by Meta is a 400B MoE model with 17B active parameters and 128 experts, delivering frontier-level multimodal performance.", - "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/646cf8084eefb026fb8fd8bc/oCTqufkdTkjyGodsx1vo1.png", - "provider": "Meta", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "1M", - "total_params": "400B", - "active_params": "17B", - "precision": "BF16 / FP8", - "model_url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Mistral Large 3", - "name": "mistral-large-3", - "description": "Mistral Large 3 is a 675B MoE model with 41B active parameters, supporting text and vision tasks.", - "model_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/634c17653d11eaedd88b314d/9OgyfKstSZtbmsmuG8MbU.png", - "provider": "Mistral AI", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "675B", - "active_params": "41B", - "precision": "FP8 / NVFP4", - "model_url": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "MiMo-V2-Flash", - "name": "mimo-v2-flash", - "description": "MiMo-V2-Flash by Xiaomi is a 309B MoE reasoning model with 15B active parameters.", - "model_id": "XiaomiMiMo/MiMo-V2-Flash", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/680cb7d1233834890a64acee/5w_4aLfF-7MAyaIPOV498.jpeg", - "provider": "Xiaomi", - "modalities": [ - "LLM" - ], - "context_window": "256K", - "total_params": "309B", - "active_params": "15B", - "precision": "FP8", - "model_url": "https://huggingface.co/XiaomiMiMo/MiMo-V2-Flash", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-Coder-480B-A35B", - "name": "qwen3-coder-480b-a35b", - "description": "Qwen3-Coder-480B-A35B by Alibaba is a 480B MoE model with 35B active parameters, optimized for code generation with hybrid reasoning.", - "model_id": "Qwen/Qwen3-Coder-480B-A35B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM" - ], - "context_window": "262K", - "total_params": "480B", - "active_params": "35B", - "precision": "FP8", - "model_url": "https://huggingface.co/Qwen/Qwen3-Coder-480B-A35B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "gpt-oss-120b", - "name": "gpt-oss-120b", - "description": "gpt-oss-120b by OpenAI is a 117B MoE model with 5.1B active parameters and 128 experts, featuring reasoning capabilities.", - "model_id": "openai/gpt-oss-120b", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/68783facef79a05727260de3/UPX5RQxiPGA-ZbBmArIKq.png", - "provider": "OpenAI", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "117B", - "active_params": "5.1B", - "precision": "MXFP4", - "model_url": "https://huggingface.co/openai/gpt-oss-120b", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": true - }, - { - "display_name": "Llama 3.1 405B", - "name": "llama-3-1-405b", - "description": "Llama 3.1 405B by Meta is a dense 405B parameter model for text generation.", - "model_id": "meta-llama/Meta-Llama-3.1-405B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/646cf8084eefb026fb8fd8bc/oCTqufkdTkjyGodsx1vo1.png", - "provider": "Meta", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "405B", - "active_params": "405B", - "precision": "BF16 / FP8", - "model_url": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Llama 4 Scout", - "name": "llama-4-scout", - "description": "Llama 4 Scout by Meta is a 109B MoE model with 17B active parameters and 16 experts, supporting text and vision with a 10M context window.", - "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/646cf8084eefb026fb8fd8bc/oCTqufkdTkjyGodsx1vo1.png", - "provider": "Meta", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "10M", - "total_params": "109B", - "active_params": "17B", - "precision": "BF16 / INT4", - "model_url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "DeepSeek R1", - "name": "deepseek-r1", - "description": "DeepSeek R1 (January 2025) is a 671B MoE reasoning model with 37B active parameters.", - "model_id": "deepseek-ai/DeepSeek-R1", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/6538815d1bdb3c40db94fbfa/xMBly9PUMphrFVMxLX4kq.png", - "provider": "DeepSeek", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "671B", - "active_params": "37B", - "precision": "FP8 / BF16", - "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3 30B-A3B", - "name": "qwen3-30b-a3b", - "description": "Qwen3 30B-A3B by Alibaba is a 30.5B MoE model with 3.3B active parameters, featuring hybrid reasoning.", - "model_id": "Qwen/Qwen3-30B-A3B", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM" - ], - "context_window": "262K", - "total_params": "30.5B", - "active_params": "3.3B", - "precision": "FP8 / AWQ", - "model_url": "https://huggingface.co/Qwen/Qwen3-30B-A3B", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "EXAONE 4.0 32B", - "name": "exaone-4-0-32b", - "description": "EXAONE 4.0 32B by LG AI Research is a dense 32B parameter model with hybrid reasoning capabilities.", - "model_id": "LGAI-EXAONE/EXAONE-4.0-32B", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/66a899a72f11aaf66001a8dc/UfdrP3GMo9pNT62BaMnhw.png", - "provider": "LG AI Research", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "32B", - "active_params": "32B", - "precision": "BF16 / AWQ", - "model_url": "https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-32B", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-Omni-30B-A3B", - "name": "qwen3-omni-30b-a3b", - "description": "Qwen3-Omni-30B-A3B by Alibaba is a 30B omni-modal MoE model with 3B active parameters, supporting text, audio, vision, and video.", - "model_id": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM", - "Audio", - "Vision" - ], - "context_window": "262K", - "total_params": "30B", - "active_params": "3B", - "precision": "BF16", - "model_url": "https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Ministral 14B", - "name": "ministral-14b", - "description": "Ministral 14B by Mistral AI is a dense 14B parameter model supporting text and vision with reasoning.", - "model_id": "mistralai/Ministral-3-14B-Instruct-2512", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/634c17653d11eaedd88b314d/9OgyfKstSZtbmsmuG8MbU.png", - "provider": "Mistral AI", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "14B", - "active_params": "14B", - "precision": "FP8 / BF16", - "model_url": "https://huggingface.co/mistralai/Ministral-3-14B-Instruct-2512", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "gpt-oss-20b", - "name": "gpt-oss-20b", - "description": "gpt-oss-20b by OpenAI is a 21B MoE model with 3.6B active parameters and 32 experts, featuring reasoning capabilities.", - "model_id": "openai/gpt-oss-20b", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/68783facef79a05727260de3/UPX5RQxiPGA-ZbBmArIKq.png", - "provider": "OpenAI", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "21B", - "active_params": "3.6B", - "precision": "MXFP4", - "model_url": "https://huggingface.co/openai/gpt-oss-20b", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Gemma 3 12B", - "name": "gemma-3-12b", - "description": "Gemma 3 12B by Google DeepMind is a dense 12B parameter model supporting text and vision.", - "model_id": "google/gemma-3-12b-it", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/5dd96eb166059660ed1ee413/WtA3YYitedOr9n02eHfJe.png", - "provider": "Google DeepMind", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "128K", - "total_params": "12B", - "active_params": "12B", - "precision": "BF16 / QAT-INT4", - "model_url": "https://huggingface.co/google/gemma-3-12b-it", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Phi-4", - "name": "phi-4", - "description": "Phi-4 by Microsoft is a dense 14B parameter model for text generation.", - "model_id": "microsoft/phi-4", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/1583646260758-5e64858c87403103f9f1055d.png", - "provider": "Microsoft", - "modalities": [ - "LLM" - ], - "context_window": "16K", - "total_params": "14B", - "active_params": "14B", - "precision": "BF16", - "model_url": "https://huggingface.co/microsoft/phi-4", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Nemotron 3 Nano", - "name": "nemotron-3-nano", - "description": "Nemotron 3 Nano by NVIDIA is a 31.6B MoE model with 3.2B active parameters, featuring reasoning and a 1M context window.", - "model_id": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/1613114437487-60262a8e0703121c822a80b6.png", - "provider": "NVIDIA", - "modalities": [ - "LLM" - ], - "context_window": "1M", - "total_params": "31.6B", - "active_params": "3.2B", - "precision": "FP8 / NVFP4", - "model_url": "https://huggingface.co/nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Llama 3.3 70B", - "name": "llama-3-3-70b", - "description": "Llama 3.3 70B by Meta is a dense 70B parameter model for text generation.", - "model_id": "meta-llama/Llama-3.3-70B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/646cf8084eefb026fb8fd8bc/oCTqufkdTkjyGodsx1vo1.png", - "provider": "Meta", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "70B", - "active_params": "70B", - "precision": "BF16 / FP8", - "model_url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen2.5 72B", - "name": "qwen2-5-72b", - "description": "Qwen2.5 72B by Alibaba is a dense 72B parameter model for text generation.", - "model_id": "Qwen/Qwen2.5-72B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "72B", - "active_params": "72B", - "precision": "BF16 / FP8", - "model_url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Mistral Small 3.1 24B", - "name": "mistral-small-3-1-24b", - "description": "Mistral Small 3.1 by Mistral AI is a dense 24B parameter model supporting text and vision.", - "model_id": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/634c17653d11eaedd88b314d/9OgyfKstSZtbmsmuG8MbU.png", - "provider": "Mistral AI", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "128K", - "total_params": "24B", - "active_params": "24B", - "precision": "BF16 / FP8", - "model_url": "https://huggingface.co/mistralai/Mistral-Small-3.1-24B-Instruct-2503", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Gemma 3 27B", - "name": "gemma-3-27b", - "description": "Gemma 3 27B by Google DeepMind is a dense 27B parameter model supporting text and vision.", - "model_id": "google/gemma-3-27b-it", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/5dd96eb166059660ed1ee413/WtA3YYitedOr9n02eHfJe.png", - "provider": "Google DeepMind", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "128K", - "total_params": "27B", - "active_params": "27B", - "precision": "BF16 / QAT-INT4", - "model_url": "https://huggingface.co/google/gemma-3-27b-it", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "DeepSeek R1 Distill Llama 70B", - "name": "deepseek-r1-distill-llama-70b", - "description": "DeepSeek R1 Distill Llama 70B is a dense 70B parameter distilled reasoning model.", - "model_id": "deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/6538815d1bdb3c40db94fbfa/xMBly9PUMphrFVMxLX4kq.png", - "provider": "DeepSeek", - "modalities": [ - "LLM" - ], - "context_window": "128K", - "total_params": "70B", - "active_params": "70B", - "precision": "BF16", - "model_url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Llama-70B", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-VL-30B-A3B", - "name": "qwen3-vl-30b-a3b", - "description": "Qwen3-VL-30B-A3B by Alibaba is a 30B MoE vision-language model with 3B active parameters, supporting text, vision, and video with reasoning.", - "model_id": "Qwen/Qwen3-VL-30B-A3B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "30B", - "active_params": "3B", - "precision": "BF16", - "model_url": "https://huggingface.co/Qwen/Qwen3-VL-30B-A3B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-VL-8B", - "name": "qwen3-vl-8b", - "description": "Qwen3-VL-8B by Alibaba is a dense 8B vision-language model supporting text, vision, and video with reasoning.", - "model_id": "Qwen/Qwen3-VL-8B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "8B", - "active_params": "8B", - "precision": "BF16", - "model_url": "https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - }, - { - "display_name": "Qwen3-VL-4B", - "name": "qwen3-vl-4b", - "description": "Qwen3-VL-4B by Alibaba is a dense 4B vision-language model supporting text, vision, and video with reasoning.", - "model_id": "Qwen/Qwen3-VL-4B-Instruct", - "logo_url": "https://cdn-avatars.huggingface.co/v1/production/uploads/620760a26e3b7210c2ff1943/-s1gyJfvbE1RgO5iBeNOi.png", - "provider": "Alibaba", - "modalities": [ - "LLM", - "Vision" - ], - "context_window": "256K", - "total_params": "4B", - "active_params": "4B", - "precision": "BF16", - "model_url": "https://huggingface.co/Qwen/Qwen3-VL-4B-Instruct", - "pricing": { - "cached_input": "0", - "input": "0", - "output": "0" - }, - "isLive": false, - "isNew": false, - "isTrending": false - } -] \ No newline at end of file From 2c200f798729119edcb7cbb3613ba6f038607741 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:09:40 -0500 Subject: [PATCH 10/20] debug: add diff field logging to identify idempotency issue --- scripts/fetch-models.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index ffc6fcd..2dd4a5c 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -114,14 +114,17 @@ export function diffModels(apiModels, webflowItems) { continue; } - const hasChanges = Object.keys(model.fields).some((key) => { + const changedFields = Object.keys(model.fields).filter((key) => { if (SKIP_DIFF_FIELDS.has(key)) return false; const apiVal = model.fields[key]; const wfVal = existing.fieldData[key]; return JSON.stringify(apiVal) !== JSON.stringify(wfVal); }); - if (hasChanges) { + if (changedFields.length > 0) { + for (const key of changedFields) { + console.log(` [diff] ${model.slug}.${key}: API=${JSON.stringify(model.fields[key])} WF=${JSON.stringify(existing.fieldData[key])}`); + } toUpdate.push({ id: existing.id, fieldData: model.fields }); } else { unchanged++; From c11860ccb36d46f2f90a0d379303809ea32d9186 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:11:50 -0500 Subject: [PATCH 11/20] fix: treat empty string and undefined as equivalent in diff --- scripts/fetch-models.js | 8 +++----- tests/fetch-models/diff.test.js | 14 ++++++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index 2dd4a5c..35fccd8 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -114,17 +114,15 @@ export function diffModels(apiModels, webflowItems) { continue; } - const changedFields = Object.keys(model.fields).filter((key) => { + const hasChanges = Object.keys(model.fields).some((key) => { if (SKIP_DIFF_FIELDS.has(key)) return false; const apiVal = model.fields[key]; const wfVal = existing.fieldData[key]; + if ((apiVal === '' || apiVal == null) && (wfVal === '' || wfVal == null)) return false; return JSON.stringify(apiVal) !== JSON.stringify(wfVal); }); - if (changedFields.length > 0) { - for (const key of changedFields) { - console.log(` [diff] ${model.slug}.${key}: API=${JSON.stringify(model.fields[key])} WF=${JSON.stringify(existing.fieldData[key])}`); - } + if (hasChanges) { toUpdate.push({ id: existing.id, fieldData: model.fields }); } else { unchanged++; diff --git a/tests/fetch-models/diff.test.js b/tests/fetch-models/diff.test.js index c931d99..cf825f8 100644 --- a/tests/fetch-models/diff.test.js +++ b/tests/fetch-models/diff.test.js @@ -82,4 +82,18 @@ describe('diffModels', () => { assert.equal(result.unchanged, 1); assert.equal(result.toUpdate.length, 0); }); + + it('treats empty string and undefined as equivalent', () => { + const apiModels = [ + makeApiModel('sparse-model', { name: 'Sparse', description: '', provider: '' }), + ]; + const webflowItems = [ + makeWfItem('wf-id-5', 'sparse-model', { name: 'Sparse' }), + ]; + + const result = diffModels(apiModels, webflowItems); + + assert.equal(result.unchanged, 1); + assert.equal(result.toUpdate.length, 0); + }); }); From 8e490577d42492522563c3f63cc200f1f329d5d1 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:18:25 -0500 Subject: [PATCH 12/20] feat: add environment toggle, dry run mode, rename workflow - Rename workflow to Sync Model Library - Add environment choice input (test/production) with TEST_ and PROD_ prefixed secrets/vars - Add dry run checkbox that shows changes without pushing - Scheduled runs default to test environment --- .github/workflows/fetch-models.yml | 19 ++++++++++++++--- scripts/fetch-models.js | 34 +++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/.github/workflows/fetch-models.yml b/.github/workflows/fetch-models.yml index 190428c..94a97b1 100644 --- a/.github/workflows/fetch-models.yml +++ b/.github/workflows/fetch-models.yml @@ -1,9 +1,21 @@ -name: Fetch Model Garden +name: Sync Model Library on: schedule: - cron: '0 0 * * *' workflow_dispatch: + inputs: + environment: + description: 'Target environment' + type: choice + options: + - test + - production + default: 'test' + dry_run: + description: 'Dry run (show changes without pushing to Webflow)' + type: boolean + default: false jobs: sync: @@ -21,5 +33,6 @@ jobs: MODULAR_CLOUD_API_TOKEN: ${{ secrets.MODULAR_CLOUD_API_TOKEN }} MODULAR_CLOUD_ORG: ${{ vars.MODULAR_CLOUD_ORG }} MODULAR_CLOUD_BASE_URL: ${{ vars.MODULAR_CLOUD_BASE_URL }} - WEBFLOW_API_TOKEN: ${{ secrets.TEST_WEBFLOW_API_TOKEN }} - WEBFLOW_SITE_ID: ${{ vars.TEST_WEBFLOW_SITE_ID }} + WEBFLOW_API_TOKEN: ${{ inputs.environment == 'production' && secrets.PROD_WEBFLOW_API_TOKEN || secrets.TEST_WEBFLOW_API_TOKEN }} + WEBFLOW_SITE_ID: ${{ inputs.environment == 'production' && vars.PROD_WEBFLOW_SITE_ID || vars.TEST_WEBFLOW_SITE_ID }} + DRY_RUN: ${{ inputs.dry_run || 'false' }} diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index 35fccd8..20fbe8c 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -7,7 +7,8 @@ const isMain = process.argv[1] === __filename; // -- Environment variables -- const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; -const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID } = process.env; +const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID, DRY_RUN } = process.env; +const dryRun = DRY_RUN === 'true'; let wf; if (isMain) { @@ -167,6 +168,10 @@ async function resolveLogo(model) { const buffer = Buffer.from(payload, 'base64'); if (buffer.length === 0) return null; + if (dryRun) { + console.log(`[dry run] Would upload logo for: ${name}`); + return { url: 'dry-run-placeholder', alt: `${display_name || name} logo` }; + } console.log(`Uploading logo for: ${name}`); const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${name}${ext}`, buffer); return { url: assetUrl, alt: `${display_name || name} logo` }; @@ -196,6 +201,9 @@ async function syncCategories(models, categoriesCollectionId) { const slug = modality.toLowerCase(); if (existingBySlug.has(slug)) { categoryMap[slug] = existingBySlug.get(slug); + } else if (dryRun) { + console.log(`[dry run] Would create category: ${modality}`); + categoryMap[slug] = `dry-run-${slug}`; } else { console.log(`Creating category: ${modality}`); const result = await wf.createItems(categoriesCollectionId, [{ name: modality, slug }]); @@ -211,6 +219,8 @@ async function syncCategories(models, categoriesCollectionId) { // -- Main -- async function main() { + if (dryRun) console.log('=== DRY RUN MODE — no changes will be pushed to Webflow ===\n'); + // Phase 1: Fetch console.log('Fetching models from Modular Cloud API...'); const modelGarden = await fetchModelGarden(); @@ -242,6 +252,28 @@ async function main() { const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); // Phase 3: Sync + if (dryRun) { + if (toCreate.length > 0) { + console.log(`[dry run] Would create ${toCreate.length} models:`); + for (const fields of toCreate) { + console.log(` ${fields.slug}`); + } + } + if (toUpdate.length > 0) { + console.log(`[dry run] Would update ${toUpdate.length} models:`); + for (const item of toUpdate) { + console.log(` ${item.fieldData.slug}`); + } + } + if (toDelete.length > 0) { + console.log(`[dry run] Would delete ${toDelete.length} models`); + } + console.log( + `\n[dry run] Summary — Create: ${toCreate.length}, Update: ${toUpdate.length}, Delete: ${toDelete.length}, Unchanged: ${unchanged}` + ); + return; + } + const publishIds = []; if (toCreate.length > 0) { From b5e362c7fe3a1e851528df53a595e9a7f57b5542 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:21:39 -0500 Subject: [PATCH 13/20] feat: default cron runs to production environment --- .github/workflows/fetch-models.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fetch-models.yml b/.github/workflows/fetch-models.yml index 94a97b1..d2d7788 100644 --- a/.github/workflows/fetch-models.yml +++ b/.github/workflows/fetch-models.yml @@ -33,6 +33,6 @@ jobs: MODULAR_CLOUD_API_TOKEN: ${{ secrets.MODULAR_CLOUD_API_TOKEN }} MODULAR_CLOUD_ORG: ${{ vars.MODULAR_CLOUD_ORG }} MODULAR_CLOUD_BASE_URL: ${{ vars.MODULAR_CLOUD_BASE_URL }} - WEBFLOW_API_TOKEN: ${{ inputs.environment == 'production' && secrets.PROD_WEBFLOW_API_TOKEN || secrets.TEST_WEBFLOW_API_TOKEN }} - WEBFLOW_SITE_ID: ${{ inputs.environment == 'production' && vars.PROD_WEBFLOW_SITE_ID || vars.TEST_WEBFLOW_SITE_ID }} + WEBFLOW_API_TOKEN: ${{ inputs.environment == 'test' && secrets.TEST_WEBFLOW_API_TOKEN || secrets.PROD_WEBFLOW_API_TOKEN }} + WEBFLOW_SITE_ID: ${{ inputs.environment == 'test' && vars.TEST_WEBFLOW_SITE_ID || vars.PROD_WEBFLOW_SITE_ID }} DRY_RUN: ${{ inputs.dry_run || 'false' }} From bad54749615f88014355fa75f4a80d34689a6c44 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:37:04 -0500 Subject: [PATCH 14/20] fix: align field slugs with production schema - logo-image -> logo - categories -> modalities - models-categories -> models-category - description: remove

wrapping (PlainText, not RichText) --- scripts/fetch-models.js | 10 +++++----- tests/fetch-models/diff.test.js | 6 +++--- tests/fetch-models/transform.test.js | 10 +++++----- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index 20fbe8c..3e9791c 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -76,8 +76,8 @@ export function toWebflowFields(model, modalities, categoryMap, logoField) { slug: model.name, 'display-name': model.display_name || '', 'model-id': model.model_id || '', - 'logo-image': logoField, - description: model.description ? `

${model.description}

` : '', + logo: logoField, + description: model.description || '', provider: model.provider || '', 'context-window': model.context_window || '', 'total-params': model.total_params || '', @@ -87,13 +87,13 @@ export function toWebflowFields(model, modalities, categoryMap, logoField) { live: model.isLive, new: model.isNew, trending: model.isTrending, - categories: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), + modalities: modalities.map((m) => categoryMap[m.toLowerCase()]).filter(Boolean), }; } // -- Diff -- -const SKIP_DIFF_FIELDS = new Set(['logo-image', 'slug']); +const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); export function diffModels(apiModels, webflowItems) { const wfBySlug = new Map(); @@ -229,7 +229,7 @@ async function main() { // Discover collections const collections = await wf.getCollections(WEBFLOW_SITE_ID); - const categoriesCol = wf.findCollectionBySlug(collections, 'models-categories'); + const categoriesCol = wf.findCollectionBySlug(collections, 'models-category'); const modelsCol = wf.findCollectionBySlug(collections, 'models'); // Sync categories diff --git a/tests/fetch-models/diff.test.js b/tests/fetch-models/diff.test.js index cf825f8..0dc85d7 100644 --- a/tests/fetch-models/diff.test.js +++ b/tests/fetch-models/diff.test.js @@ -63,17 +63,17 @@ describe('diffModels', () => { assert.equal(result.toDelete.length, 0); }); - it('ignores logo-image field differences in comparison', () => { + it('ignores logo field differences in comparison', () => { const apiModels = [ makeApiModel('logo-model', { name: 'Logo Model', - 'logo-image': { fileId: 'new-file-id', url: 'https://cdn.example.com/new.png' }, + 'logo': { fileId: 'new-file-id', url: 'https://cdn.example.com/new.png' }, }), ]; const webflowItems = [ makeWfItem('wf-id-4', 'logo-model', { name: 'Logo Model', - 'logo-image': { fileId: 'old-file-id', url: 'https://cdn.example.com/old.png' }, + 'logo': { fileId: 'old-file-id', url: 'https://cdn.example.com/old.png' }, }), ]; diff --git a/tests/fetch-models/transform.test.js b/tests/fetch-models/transform.test.js index 6540783..7dd8fdd 100644 --- a/tests/fetch-models/transform.test.js +++ b/tests/fetch-models/transform.test.js @@ -29,8 +29,8 @@ describe('toWebflowFields', () => { assert.equal(result.slug, 'my-model'); assert.equal(result['display-name'], 'My Model'); assert.equal(result['model-id'], 'org/my-model'); - assert.deepEqual(result['logo-image'], logoField); - assert.equal(result.description, '

A great model

'); + assert.deepEqual(result.logo, logoField); + assert.equal(result.description, 'A great model'); assert.equal(result.provider, 'Acme'); assert.equal(result['context-window'], 8192); assert.equal(result['total-params'], '70B'); @@ -40,7 +40,7 @@ describe('toWebflowFields', () => { assert.equal(result.live, true); assert.equal(result.new, false); assert.equal(result.trending, true); - assert.deepEqual(result.categories, ['id-text', 'id-image']); + assert.deepEqual(result.modalities, ['id-text', 'id-image']); }); it('handles undefined optional fields with empty string defaults', () => { @@ -61,7 +61,7 @@ describe('toWebflowFields', () => { assert.equal(result['active-params'], ''); assert.equal(result.precision, ''); assert.equal(result['model-url'], ''); - assert.deepEqual(result.categories, []); + assert.deepEqual(result.modalities, []); }); it('falls back to name when display_name is falsy', () => { @@ -91,6 +91,6 @@ describe('toWebflowFields', () => { const result = toWebflowFields(model, modalities, categoryMap, null); - assert.deepEqual(result.categories, ['id-text', 'id-audio']); + assert.deepEqual(result.modalities, ['id-text', 'id-audio']); }); }); From 449b662841cd31cdfa8ad6dd29ef9ced6bed29ef Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:41:00 -0500 Subject: [PATCH 15/20] feat: use live CMS endpoints, remove publish step All create/update/delete operations now use /items/live endpoints which publish immediately, eliminating the separate publish step. --- scripts/fetch-models.js | 12 +----------- scripts/webflow-api.js | 27 +++------------------------ 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index 3e9791c..ec7b245 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -209,7 +209,6 @@ async function syncCategories(models, categoriesCollectionId) { const result = await wf.createItems(categoriesCollectionId, [{ name: modality, slug }]); const newItem = result.items[0]; categoryMap[slug] = newItem.id; - await wf.publishItems(categoriesCollectionId, [newItem.id]); } } @@ -274,15 +273,12 @@ async function main() { return; } - const publishIds = []; - if (toCreate.length > 0) { console.log(`Creating ${toCreate.length} models...`); for (const fields of toCreate) { console.log(` Creating: ${fields.slug}`); } - const created = await wf.createItems(modelsCol.id, toCreate); - publishIds.push(...created.items.map((item) => item.id)); + await wf.createItems(modelsCol.id, toCreate); } if (toUpdate.length > 0) { @@ -291,7 +287,6 @@ async function main() { console.log(` Updating: ${item.fieldData.slug}`); } await wf.updateItems(modelsCol.id, toUpdate); - publishIds.push(...toUpdate.map((item) => item.id)); } if (toDelete.length > 0) { @@ -299,11 +294,6 @@ async function main() { await wf.deleteItems(modelsCol.id, toDelete); } - if (publishIds.length > 0) { - console.log(`Publishing ${publishIds.length} items...`); - await wf.publishItems(modelsCol.id, publishIds); - } - // Summary console.log( `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js index e97aa3c..d996ebb 100644 --- a/scripts/webflow-api.js +++ b/scripts/webflow-api.js @@ -80,7 +80,7 @@ function createClient(apiToken) { const body = { items: batch.map((fieldData) => ({ fieldData })), }; - const data = await webflowFetch(`/collections/${collectionId}/items`, { + const data = await webflowFetch(`/collections/${collectionId}/items/live`, { method: 'POST', body: JSON.stringify(body), }); @@ -96,7 +96,7 @@ function createClient(apiToken) { const allUpdated = []; for (const batch of batches) { - const data = await webflowFetch(`/collections/${collectionId}/items`, { + const data = await webflowFetch(`/collections/${collectionId}/items/live`, { method: 'PATCH', body: JSON.stringify({ items: batch }), }); @@ -111,33 +111,13 @@ function createClient(apiToken) { const batches = chunk(itemIds, 100); for (const batch of batches) { - await webflowFetch(`/collections/${collectionId}/items`, { + await webflowFetch(`/collections/${collectionId}/items/live`, { method: 'DELETE', body: JSON.stringify({ itemIds: batch }), }); } } - async function publishItems(collectionId, itemIds) { - const batches = chunk(itemIds, 100); - const allPublished = []; - - for (const batch of batches) { - const data = await webflowFetch( - `/collections/${collectionId}/items/publish`, - { - method: 'POST', - body: JSON.stringify({ itemIds: batch }), - } - ); - if (data?.publishedItemIds) { - allPublished.push(...data.publishedItemIds); - } - } - - return { publishedItemIds: allPublished }; - } - async function uploadAsset(siteId, fileName, fileBuffer) { // Step 1: Compute MD5 hash const fileHash = createHash('md5').update(fileBuffer).digest('hex'); @@ -181,7 +161,6 @@ function createClient(apiToken) { createItems, updateItems, deleteItems, - publishItems, uploadAsset, }; } From c9f327045a33fce81647795d4cf019bf2024c2cd Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:45:49 -0500 Subject: [PATCH 16/20] feat: auto-detect live endpoint support, fall back to staged + publish Sites that have never been published return 404 on /items/live. The client now probes with a GET request and caches the result per collection. If live isn't available, uses staged endpoints and publishes after create/update. --- scripts/webflow-api.js | 57 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js index d996ebb..3d487ed 100644 --- a/scripts/webflow-api.js +++ b/scripts/webflow-api.js @@ -37,6 +37,29 @@ function createClient(apiToken) { return response.json(); } + // Detect whether /items/live endpoints are available (they 404 on + // sites that have never been published). Cache the result per collection. + const liveSupported = new Map(); + + async function supportsLive(collectionId) { + if (liveSupported.has(collectionId)) return liveSupported.get(collectionId); + + try { + await webflowFetch( + `/collections/${collectionId}/items/live?limit=1` + ); + liveSupported.set(collectionId, true); + return true; + } catch (err) { + if (err.message.includes('404')) { + console.log('Live endpoints not available for this site, using staged + publish'); + liveSupported.set(collectionId, false); + return false; + } + throw err; + } + } + async function getCollections(siteId) { const data = await webflowFetch(`/sites/${siteId}/collections`); return data.collections; @@ -72,7 +95,19 @@ function createClient(apiToken) { return allItems; } + async function publishItems(collectionId, itemIds) { + const batches = chunk(itemIds, 100); + for (const batch of batches) { + await webflowFetch(`/collections/${collectionId}/items/publish`, { + method: 'POST', + body: JSON.stringify({ itemIds: batch }), + }); + } + } + async function createItems(collectionId, fieldDataArray) { + const live = await supportsLive(collectionId); + const suffix = live ? '/live' : ''; const batches = chunk(fieldDataArray, 100); const allCreated = []; @@ -80,7 +115,7 @@ function createClient(apiToken) { const body = { items: batch.map((fieldData) => ({ fieldData })), }; - const data = await webflowFetch(`/collections/${collectionId}/items/live`, { + const data = await webflowFetch(`/collections/${collectionId}/items${suffix}`, { method: 'POST', body: JSON.stringify(body), }); @@ -88,15 +123,21 @@ function createClient(apiToken) { allCreated.push(...created); } + if (!live && allCreated.length > 0) { + await publishItems(collectionId, allCreated.map((item) => item.id)); + } + return { items: allCreated }; } async function updateItems(collectionId, itemsArray) { + const live = await supportsLive(collectionId); + const suffix = live ? '/live' : ''; const batches = chunk(itemsArray, 100); const allUpdated = []; for (const batch of batches) { - const data = await webflowFetch(`/collections/${collectionId}/items/live`, { + const data = await webflowFetch(`/collections/${collectionId}/items${suffix}`, { method: 'PATCH', body: JSON.stringify({ items: batch }), }); @@ -104,14 +145,20 @@ function createClient(apiToken) { allUpdated.push(...updated); } + if (!live && allUpdated.length > 0) { + await publishItems(collectionId, allUpdated.map((item) => item.id)); + } + return { items: allUpdated }; } async function deleteItems(collectionId, itemIds) { + const live = await supportsLive(collectionId); + const suffix = live ? '/live' : ''; const batches = chunk(itemIds, 100); for (const batch of batches) { - await webflowFetch(`/collections/${collectionId}/items/live`, { + await webflowFetch(`/collections/${collectionId}/items${suffix}`, { method: 'DELETE', body: JSON.stringify({ itemIds: batch }), }); @@ -119,10 +166,8 @@ function createClient(apiToken) { } async function uploadAsset(siteId, fileName, fileBuffer) { - // Step 1: Compute MD5 hash const fileHash = createHash('md5').update(fileBuffer).digest('hex'); - // Step 2: Request presigned upload URL const metadata = await webflowFetch(`/sites/${siteId}/assets`, { method: 'POST', body: JSON.stringify({ fileName, fileHash }), @@ -130,7 +175,6 @@ function createClient(apiToken) { const { uploadUrl, uploadDetails } = metadata; - // Step 3: Upload file via multipart/form-data to the presigned URL const formData = new FormData(); for (const [key, value] of Object.entries(uploadDetails)) { formData.append(key, value); @@ -149,7 +193,6 @@ function createClient(apiToken) { ); } - // Step 4: Return hosted URL return metadata.hostedUrl || metadata.url || metadata.assetUrl; } From 12ccac62f2e335ad44239a82ce47fe43cefc0fee Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:51:50 -0500 Subject: [PATCH 17/20] fix: gracefully handle 404 on publish endpoint for unpublished sites --- scripts/webflow-api.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js index 3d487ed..6eea27b 100644 --- a/scripts/webflow-api.js +++ b/scripts/webflow-api.js @@ -98,10 +98,18 @@ function createClient(apiToken) { async function publishItems(collectionId, itemIds) { const batches = chunk(itemIds, 100); for (const batch of batches) { - await webflowFetch(`/collections/${collectionId}/items/publish`, { - method: 'POST', - body: JSON.stringify({ itemIds: batch }), - }); + try { + await webflowFetch(`/collections/${collectionId}/items/publish`, { + method: 'POST', + body: JSON.stringify({ itemIds: batch }), + }); + } catch (err) { + if (err.message.includes('404')) { + console.log('Publish endpoint not available — site may need to be published first'); + return; + } + throw err; + } } } From a35f0b8df28b98402a67b32346dca38a47c9e404 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:56:34 -0500 Subject: [PATCH 18/20] fix: skip logo upload for models that already have a logo in Webflow --- scripts/fetch-models.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index ec7b245..b5d98fe 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -236,18 +236,25 @@ async function main() { const categoryMap = await syncCategories(models, categoriesCol.id); console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); + // Fetch existing Webflow items (needed for diff and logo skip) + console.log('Fetching existing Webflow items...'); + const webflowItems = await wf.listCollectionItems(modelsCol.id); + const existingBySlug = new Map(); + for (const item of webflowItems) { + existingBySlug.set(item.fieldData.slug, item); + } + // Resolve logos and build field data console.log('Resolving logos and building field data...'); const apiModels = []; for (const model of models) { - const logoField = await resolveLogo(model); + const existing = existingBySlug.get(model.name); + const existingLogo = existing?.fieldData?.logo; + // Skip logo upload if the model already exists with a logo + const logoField = existingLogo ? existingLogo : await resolveLogo(model); const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); apiModels.push({ slug: model.name, fields }); } - - // Phase 2: Diff - console.log('Fetching existing Webflow items...'); - const webflowItems = await wf.listCollectionItems(modelsCol.id); const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); // Phase 3: Sync From ac2a88f0561424d1eadc6e0acff099205b192ce4 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 10:59:55 -0500 Subject: [PATCH 19/20] fix: only skip re-upload for base64 logos, pass URL logos through normally --- scripts/fetch-models.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index b5d98fe..74e66c6 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -250,8 +250,9 @@ async function main() { for (const model of models) { const existing = existingBySlug.get(model.name); const existingLogo = existing?.fieldData?.logo; - // Skip logo upload if the model already exists with a logo - const logoField = existingLogo ? existingLogo : await resolveLogo(model); + // Skip base64 logo re-upload if the model already has a logo in Webflow + const isBase64 = model.logo_url && model.logo_url.startsWith('data:'); + const logoField = isBase64 && existingLogo ? existingLogo : await resolveLogo(model); const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); apiModels.push({ slug: model.name, fields }); } From b8e39bcb49d0cfea206a2b3797bfd7ead544ce65 Mon Sep 17 00:00:00 2001 From: Bill Welense Date: Mon, 23 Mar 2026 11:08:10 -0500 Subject: [PATCH 20/20] refactor: apply composed method pattern to sync scripts fetch-models.js: - main() reads as a story: fetch, discover, sync categories, build field data, diff, apply changes - Dry-run gate in one place (main), not scattered across helpers - Renamed: transformModel -> normalizeApiModel, toWebflowFields -> buildWebflowFields, SKIP_DIFF_FIELDS -> FIELDS_MANAGED_OUTSIDE_DIFF - resolveLogo split into isUrl/isBase64DataUri/buildLogoField/ uploadBase64Logo/parseBase64DataUri - Extracted: fetchModels, discoverCollections, fetchExistingItems, buildModelFieldData, applyChanges, logDryRunSummary, logSyncSummary webflow-api.js: - Renamed live/suffix -> useLiveEndpoint/liveSuffix - Documented asset URL fallback --- scripts/fetch-models.js | 255 ++++++++++++++++----------- scripts/webflow-api.js | 23 +-- tests/fetch-models/transform.test.js | 12 +- 3 files changed, 170 insertions(+), 120 deletions(-) diff --git a/scripts/fetch-models.js b/scripts/fetch-models.js index 74e66c6..a04dc0a 100644 --- a/scripts/fetch-models.js +++ b/scripts/fetch-models.js @@ -4,7 +4,7 @@ import { createClient } from './webflow-api.js'; const __filename = fileURLToPath(import.meta.url); const isMain = process.argv[1] === __filename; -// -- Environment variables -- +// -- Configuration -- const { MODULAR_CLOUD_API_TOKEN, MODULAR_CLOUD_ORG, MODULAR_CLOUD_BASE_URL } = process.env; const { WEBFLOW_API_TOKEN, WEBFLOW_SITE_ID, DRY_RUN } = process.env; @@ -46,7 +46,7 @@ async function fetchModelGarden() { return listRes.json(); } -function transformModel(model) { +function normalizeApiModel(model) { const meta = model.metadata || {}; const tags = meta.tags || []; return { @@ -68,9 +68,9 @@ function transformModel(model) { }; } -// -- Field mapping -- +// -- Webflow field mapping -- -export function toWebflowFields(model, modalities, categoryMap, logoField) { +export function buildWebflowFields(model, modalities, categoryMap, logoField) { return { name: model.display_name || model.name, slug: model.name, @@ -93,7 +93,8 @@ export function toWebflowFields(model, modalities, categoryMap, logoField) { // -- Diff -- -const SKIP_DIFF_FIELDS = new Set(['logo', 'slug']); +// Logo is resolved separately (upload vs URL); slug is identity, not content +const FIELDS_MANAGED_OUTSIDE_DIFF = new Set(['logo', 'slug']); export function diffModels(apiModels, webflowItems) { const wfBySlug = new Map(); @@ -116,7 +117,7 @@ export function diffModels(apiModels, webflowItems) { } const hasChanges = Object.keys(model.fields).some((key) => { - if (SKIP_DIFF_FIELDS.has(key)) return false; + if (FIELDS_MANAGED_OUTSIDE_DIFF.has(key)) return false; const apiVal = model.fields[key]; const wfVal = existing.fieldData[key]; if ((apiVal === '' || apiVal == null) && (wfVal === '' || wfVal == null)) return false; @@ -148,56 +149,94 @@ const MIME_TO_EXT = { 'image/webp': '.webp', }; -async function resolveLogo(model) { - const { logo_url, display_name, name } = model; +function isUrl(str) { + return str.startsWith('http'); +} - if (!logo_url) return null; +function isBase64DataUri(str) { + return str.startsWith('data:'); +} - if (logo_url.startsWith('http')) { - return { url: logo_url, alt: `${display_name || name} logo` }; - } +function buildLogoField(model) { + return { url: model.logo_url, alt: `${model.display_name || model.name} logo` }; +} - if (logo_url.startsWith('data:')) { - const match = logo_url.match(/^data:([^;]+);base64,(.+)$/s); - if (!match) return null; +function parseBase64DataUri(dataUri) { + const match = dataUri.match(/^data:([^;]+);base64,(.+)$/s); + if (!match) return null; - const [, mime, payload] = match; - const ext = MIME_TO_EXT[mime]; - if (!ext) return null; + const [, mime, payload] = match; + const ext = MIME_TO_EXT[mime]; + if (!ext) return null; - const buffer = Buffer.from(payload, 'base64'); - if (buffer.length === 0) return null; + const buffer = Buffer.from(payload, 'base64'); + if (buffer.length === 0) return null; - if (dryRun) { - console.log(`[dry run] Would upload logo for: ${name}`); - return { url: 'dry-run-placeholder', alt: `${display_name || name} logo` }; - } - console.log(`Uploading logo for: ${name}`); - const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${name}${ext}`, buffer); - return { url: assetUrl, alt: `${display_name || name} logo` }; - } + return { ext, buffer }; +} + +async function uploadBase64Logo(model) { + const parsed = parseBase64DataUri(model.logo_url); + if (!parsed) return null; + + console.log(`Uploading logo for: ${model.name}`); + const assetUrl = await wf.uploadAsset(WEBFLOW_SITE_ID, `${model.name}${parsed.ext}`, parsed.buffer); + return { url: assetUrl, alt: `${model.display_name || model.name} logo` }; +} +async function resolveLogo(model) { + const { logo_url } = model; + if (!logo_url) return null; + if (isUrl(logo_url)) return buildLogoField(model); + if (isBase64DataUri(logo_url)) return uploadBase64Logo(model); return null; } -// -- Categories sync -- +// -- Orchestration helpers -- -async function syncCategories(models, categoriesCollectionId) { - const allModalities = new Set(); +async function fetchModels() { + console.log('Fetching models from Modular Cloud API...'); + const modelGarden = await fetchModelGarden(); + const models = modelGarden.items.map(normalizeApiModel); + console.log(`Fetched ${models.length} models`); + return models; +} + +async function discoverCollections() { + const collections = await wf.getCollections(WEBFLOW_SITE_ID); + return { + categoriesCol: wf.findCollectionBySlug(collections, 'models-category'), + modelsCol: wf.findCollectionBySlug(collections, 'models'), + }; +} + +function collectUniqueModalities(models) { + const all = new Set(); for (const model of models) { if (model.modalities) { - for (const m of model.modalities) allModalities.add(m); + for (const m of model.modalities) all.add(m); } } + return all; +} - const existingItems = await wf.listCollectionItems(categoriesCollectionId); - const existingBySlug = new Map(); - for (const item of existingItems) { - existingBySlug.set(item.fieldData.slug, item.id); +function buildExistingCategoryMap(items) { + const map = new Map(); + for (const item of items) { + map.set(item.fieldData.slug, item.id); } + return map; +} + +async function syncCategories(models, categoriesCollectionId) { + console.log('Syncing categories...'); + + const needed = collectUniqueModalities(models); + const existingItems = await wf.listCollectionItems(categoriesCollectionId); + const existingBySlug = buildExistingCategoryMap(existingItems); const categoryMap = {}; - for (const modality of allModalities) { + for (const modality of needed) { const slug = modality.toLowerCase(); if (existingBySlug.has(slug)) { categoryMap[slug] = existingBySlug.get(slug); @@ -207,107 +246,117 @@ async function syncCategories(models, categoriesCollectionId) { } else { console.log(`Creating category: ${modality}`); const result = await wf.createItems(categoriesCollectionId, [{ name: modality, slug }]); - const newItem = result.items[0]; - categoryMap[slug] = newItem.id; + categoryMap[slug] = result.items[0].id; } } + console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); return categoryMap; } -// -- Main -- - -async function main() { - if (dryRun) console.log('=== DRY RUN MODE — no changes will be pushed to Webflow ===\n'); - - // Phase 1: Fetch - console.log('Fetching models from Modular Cloud API...'); - const modelGarden = await fetchModelGarden(); - const models = modelGarden.items.map(transformModel); - console.log(`Fetched ${models.length} models`); - - // Discover collections - const collections = await wf.getCollections(WEBFLOW_SITE_ID); - const categoriesCol = wf.findCollectionBySlug(collections, 'models-category'); - const modelsCol = wf.findCollectionBySlug(collections, 'models'); - - // Sync categories - console.log('Syncing categories...'); - const categoryMap = await syncCategories(models, categoriesCol.id); - console.log(`Categories ready: ${Object.keys(categoryMap).join(', ')}`); - - // Fetch existing Webflow items (needed for diff and logo skip) +async function fetchExistingItems(collectionId) { console.log('Fetching existing Webflow items...'); - const webflowItems = await wf.listCollectionItems(modelsCol.id); - const existingBySlug = new Map(); - for (const item of webflowItems) { - existingBySlug.set(item.fieldData.slug, item); + return wf.listCollectionItems(collectionId); +} + +function indexBySlug(items) { + const map = new Map(); + for (const item of items) { + map.set(item.fieldData.slug, item); } + return map; +} - // Resolve logos and build field data +async function buildModelFieldData(models, categoryMap, existingItems) { console.log('Resolving logos and building field data...'); + const existingBySlug = indexBySlug(existingItems); const apiModels = []; + for (const model of models) { const existing = existingBySlug.get(model.name); const existingLogo = existing?.fieldData?.logo; - // Skip base64 logo re-upload if the model already has a logo in Webflow - const isBase64 = model.logo_url && model.logo_url.startsWith('data:'); - const logoField = isBase64 && existingLogo ? existingLogo : await resolveLogo(model); - const fields = toWebflowFields(model, model.modalities || [], categoryMap, logoField); + const hasBase64Logo = model.logo_url && isBase64DataUri(model.logo_url); + + // Reuse existing logo for base64 sources (avoid re-upload every run) + let logoField; + if (hasBase64Logo && existingLogo) { + logoField = existingLogo; + } else if (dryRun && hasBase64Logo) { + console.log(`[dry run] Would upload logo for: ${model.name}`); + logoField = { url: 'dry-run-placeholder', alt: `${model.display_name || model.name} logo` }; + } else { + logoField = await resolveLogo(model); + } + + const fields = buildWebflowFields(model, model.modalities || [], categoryMap, logoField); apiModels.push({ slug: model.name, fields }); } - const { toCreate, toUpdate, toDelete, unchanged } = diffModels(apiModels, webflowItems); - // Phase 3: Sync - if (dryRun) { - if (toCreate.length > 0) { - console.log(`[dry run] Would create ${toCreate.length} models:`); - for (const fields of toCreate) { - console.log(` ${fields.slug}`); - } - } - if (toUpdate.length > 0) { - console.log(`[dry run] Would update ${toUpdate.length} models:`); - for (const item of toUpdate) { - console.log(` ${item.fieldData.slug}`); - } - } - if (toDelete.length > 0) { - console.log(`[dry run] Would delete ${toDelete.length} models`); - } - console.log( - `\n[dry run] Summary — Create: ${toCreate.length}, Update: ${toUpdate.length}, Delete: ${toDelete.length}, Unchanged: ${unchanged}` - ); - return; - } + return apiModels; +} +async function applyChanges(collectionId, { toCreate, toUpdate, toDelete }) { if (toCreate.length > 0) { console.log(`Creating ${toCreate.length} models...`); - for (const fields of toCreate) { - console.log(` Creating: ${fields.slug}`); - } - await wf.createItems(modelsCol.id, toCreate); + for (const fields of toCreate) console.log(` Creating: ${fields.slug}`); + await wf.createItems(collectionId, toCreate); } if (toUpdate.length > 0) { console.log(`Updating ${toUpdate.length} models...`); - for (const item of toUpdate) { - console.log(` Updating: ${item.fieldData.slug}`); - } - await wf.updateItems(modelsCol.id, toUpdate); + for (const item of toUpdate) console.log(` Updating: ${item.fieldData.slug}`); + await wf.updateItems(collectionId, toUpdate); } if (toDelete.length > 0) { console.log(`Deleting ${toDelete.length} models...`); - await wf.deleteItems(modelsCol.id, toDelete); + await wf.deleteItems(collectionId, toDelete); } +} + +function logDryRunSummary({ toCreate, toUpdate, toDelete, unchanged }) { + if (toCreate.length > 0) { + console.log(`[dry run] Would create ${toCreate.length} models:`); + for (const fields of toCreate) console.log(` ${fields.slug}`); + } + if (toUpdate.length > 0) { + console.log(`[dry run] Would update ${toUpdate.length} models:`); + for (const item of toUpdate) console.log(` ${item.fieldData.slug}`); + } + if (toDelete.length > 0) { + console.log(`[dry run] Would delete ${toDelete.length} models`); + } + console.log( + `\n[dry run] Summary — Create: ${toCreate.length}, Update: ${toUpdate.length}, Delete: ${toDelete.length}, Unchanged: ${unchanged}` + ); +} - // Summary +function logSyncSummary({ toCreate, toUpdate, toDelete, unchanged }) { console.log( `\nSync complete. Created: ${toCreate.length}, Updated: ${toUpdate.length}, Deleted: ${toDelete.length}, Unchanged: ${unchanged}` ); } +// -- Main -- + +async function main() { + if (dryRun) console.log('=== DRY RUN MODE — no changes will be pushed to Webflow ===\n'); + + const models = await fetchModels(); + const { categoriesCol, modelsCol } = await discoverCollections(); + const categoryMap = await syncCategories(models, categoriesCol.id); + const existingItems = await fetchExistingItems(modelsCol.id); + const apiModels = await buildModelFieldData(models, categoryMap, existingItems); + const changes = diffModels(apiModels, existingItems); + + if (dryRun) { + logDryRunSummary(changes); + } else { + await applyChanges(modelsCol.id, changes); + logSyncSummary(changes); + } +} + if (isMain) { main().catch((err) => { console.error(err); diff --git a/scripts/webflow-api.js b/scripts/webflow-api.js index 6eea27b..faaa607 100644 --- a/scripts/webflow-api.js +++ b/scripts/webflow-api.js @@ -114,8 +114,8 @@ function createClient(apiToken) { } async function createItems(collectionId, fieldDataArray) { - const live = await supportsLive(collectionId); - const suffix = live ? '/live' : ''; + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; const batches = chunk(fieldDataArray, 100); const allCreated = []; @@ -123,7 +123,7 @@ function createClient(apiToken) { const body = { items: batch.map((fieldData) => ({ fieldData })), }; - const data = await webflowFetch(`/collections/${collectionId}/items${suffix}`, { + const data = await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { method: 'POST', body: JSON.stringify(body), }); @@ -131,7 +131,7 @@ function createClient(apiToken) { allCreated.push(...created); } - if (!live && allCreated.length > 0) { + if (!useLiveEndpoint && allCreated.length > 0) { await publishItems(collectionId, allCreated.map((item) => item.id)); } @@ -139,13 +139,13 @@ function createClient(apiToken) { } async function updateItems(collectionId, itemsArray) { - const live = await supportsLive(collectionId); - const suffix = live ? '/live' : ''; + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; const batches = chunk(itemsArray, 100); const allUpdated = []; for (const batch of batches) { - const data = await webflowFetch(`/collections/${collectionId}/items${suffix}`, { + const data = await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { method: 'PATCH', body: JSON.stringify({ items: batch }), }); @@ -153,7 +153,7 @@ function createClient(apiToken) { allUpdated.push(...updated); } - if (!live && allUpdated.length > 0) { + if (!useLiveEndpoint && allUpdated.length > 0) { await publishItems(collectionId, allUpdated.map((item) => item.id)); } @@ -161,12 +161,12 @@ function createClient(apiToken) { } async function deleteItems(collectionId, itemIds) { - const live = await supportsLive(collectionId); - const suffix = live ? '/live' : ''; + const useLiveEndpoint = await supportsLive(collectionId); + const liveSuffix = useLiveEndpoint ? '/live' : ''; const batches = chunk(itemIds, 100); for (const batch of batches) { - await webflowFetch(`/collections/${collectionId}/items${suffix}`, { + await webflowFetch(`/collections/${collectionId}/items${liveSuffix}`, { method: 'DELETE', body: JSON.stringify({ itemIds: batch }), }); @@ -201,6 +201,7 @@ function createClient(apiToken) { ); } + // Webflow's response shape varies by API version — check all known fields return metadata.hostedUrl || metadata.url || metadata.assetUrl; } diff --git a/tests/fetch-models/transform.test.js b/tests/fetch-models/transform.test.js index 7dd8fdd..ede42d3 100644 --- a/tests/fetch-models/transform.test.js +++ b/tests/fetch-models/transform.test.js @@ -1,8 +1,8 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { toWebflowFields } from '../../scripts/fetch-models.js'; +import { buildWebflowFields } from '../../scripts/fetch-models.js'; -describe('toWebflowFields', () => { +describe('buildWebflowFields', () => { it('maps all fields correctly for a full model', () => { const model = { display_name: 'My Model', @@ -23,7 +23,7 @@ describe('toWebflowFields', () => { const categoryMap = { text: 'id-text', image: 'id-image' }; const logoField = { fileId: 'abc', url: 'https://cdn.example.com/logo.png' }; - const result = toWebflowFields(model, modalities, categoryMap, logoField); + const result = buildWebflowFields(model, modalities, categoryMap, logoField); assert.equal(result.name, 'My Model'); assert.equal(result.slug, 'my-model'); @@ -51,7 +51,7 @@ describe('toWebflowFields', () => { isNew: false, isTrending: false, }; - const result = toWebflowFields(model, [], {}, null); + const result = buildWebflowFields(model, [], {}, null); assert.equal(result['model-id'], ''); assert.equal(result.description, ''); @@ -72,7 +72,7 @@ describe('toWebflowFields', () => { isNew: false, isTrending: false, }; - const result = toWebflowFields(model, [], {}, null); + const result = buildWebflowFields(model, [], {}, null); assert.equal(result.name, 'fallback-model'); assert.equal(result['display-name'], ''); @@ -89,7 +89,7 @@ describe('toWebflowFields', () => { const modalities = ['Text', 'Video', 'Audio']; const categoryMap = { text: 'id-text', audio: 'id-audio' }; - const result = toWebflowFields(model, modalities, categoryMap, null); + const result = buildWebflowFields(model, modalities, categoryMap, null); assert.deepEqual(result.modalities, ['id-text', 'id-audio']); });