Optimize Airtable syncing

plugins/airtable/src/App.css

@@ -225,3 +225,19 @@ select:not(:disabled) {
     flex: 1;
     width: 100%;
 }
+
+.note {
+    display: flex;
+    flex-direction: row;
+    align-items: start;
+    justify-content: left;
+    gap: 14px;
+    width: 100%;
+    padding-left: 4px;
+    color: var(--framer-color-text-tertiary);
+}
+
+.note svg {
+    margin-top: 3px;
+    flex-shrink: 0;
+}

plugins/airtable/src/App.tsx

@@ -17,9 +17,16 @@ interface AppProps {
     previousBaseId: string | null
     previousTableId: string | null
     previousSlugFieldId: string | null
+    previousLastSynced: string | null
 }
 
-export function App({ collection, previousBaseId, previousTableId, previousSlugFieldId }: AppProps) {
+export function App({
+    collection,
+    previousBaseId,
+    previousTableId,
+    previousSlugFieldId,
+    previousLastSynced,
+}: AppProps) {
     const [dataSource, setDataSource] = useState<DataSource | null>(null)
     const [isLoadingDataSource, setIsLoadingDataSource] = useState(Boolean(previousBaseId && previousTableId))
     const [noTableAccess, setNoTableAccess] = useState(false)
@@ -97,5 +104,12 @@ export function App({ collection, previousBaseId, previousTableId, previousSlugF
         return <SelectDataSource collection={collection} onSelectDataSource={setDataSource} />
     }
 
-    return <FieldMapping collection={collection} dataSource={dataSource} initialSlugFieldId={previousSlugFieldId} />
+    return (
+        <FieldMapping
+            collection={collection}
+            dataSource={dataSource}
+            initialSlugFieldId={previousSlugFieldId}
+            previousLastSynced={previousLastSynced}
+        />
+    )
 }

plugins/airtable/src/FieldMapping.tsx

@@ -2,7 +2,7 @@ import type { Field, ManagedCollection, ManagedCollectionField, ManagedCollectio
 import { FramerPluginClosedError, framer, useIsAllowedTo } from "framer-plugin"
 import { memo, useEffect, useMemo, useState } from "react"
 import type { DataSource } from "./data"
-import { mergeFieldsWithExistingFields, syncCollection, syncMethods } from "./data"
+import { isFullLastModifiedTimeField, mergeFieldsWithExistingFields, syncCollection, syncMethods } from "./data"
 import type { PossibleField } from "./fields"
 import { isCollectionReference } from "./utils"
 
@@ -147,9 +147,10 @@ interface FieldMappingProps {
     collection: ManagedCollection
     dataSource: DataSource
     initialSlugFieldId: string | null
+    previousLastSynced: string | null
 }
 
-export function FieldMapping({ collection, dataSource, initialSlugFieldId }: FieldMappingProps) {
+export function FieldMapping({ collection, dataSource, initialSlugFieldId, previousLastSynced }: FieldMappingProps) {
     const [status, setStatus] = useState<"mapping-fields" | "loading-fields" | "syncing-collection">(
         initialSlugFieldId ? "loading-fields" : "mapping-fields"
     )
@@ -165,6 +166,8 @@ export function FieldMapping({ collection, dataSource, initialSlugFieldId }: Fie
     const [fields, setFields] = useState(initialManagedCollectionFields)
     const [ignoredFieldIds, setIgnoredFieldIds] = useState(initialFieldIds)
 
+    const hasLastModifiedTimeField = dataSource.fields.some(isFullLastModifiedTimeField)
+
     // Create a map of field IDs to names for efficient lookup
     const originalFieldNameMap = useMemo(
         () => new Map(dataSource.fields.map(field => [field.id, field.name])),
@@ -310,8 +313,17 @@ export function FieldMapping({ collection, dataSource, initialSlugFieldId }: Fie
                             field.collectionId !== ""
                     )
 
+                const syncStartedAtDate = new Date().toISOString()
+
                 await collection.setFields(processFields(fieldsToSync))
-                await syncCollection(collection, dataSource, fieldsToSync, selectedSlugField.id)
+                await syncCollection(
+                    collection,
+                    dataSource,
+                    fieldsToSync,
+                    selectedSlugField.id,
+                    previousLastSynced,
+                    syncStartedAtDate
+                )
                 framer.closePlugin("Synchronization successful", {
                     variant: "success",
                 })
@@ -347,6 +359,21 @@ export function FieldMapping({ collection, dataSource, initialSlugFieldId }: Fie
         <form className="framer-hide-scrollbar mapping" onSubmit={handleSubmit}>
             <hr className="sticky-top" />
 
+            {!hasLastModifiedTimeField && (
+                <>
+                    <div className="note">
+                        <svg xmlns="http://www.w3.org/2000/svg" width="12" height="12">
+                            <path
+                                d="M6 0a6 6 0 1 1 0 12A6 6 0 0 1 6 0Zm0 4a1 1 0 1 0 0-2 1 1 0 0 0 0 2ZM5 9a1 1 0 0 0 2 0V6a1 1 0 0 0-2 0Z"
+                                fill="currentColor"
+                            ></path>
+                        </svg>
+                        <p>Add a “Last Modified Time” column in Airtable to sync faster.</p>
+                    </div>
+                    <hr />
+                </>
+            )}
+
             <label className="slug-field" htmlFor="slugField">
                 <span>Slug Field</span>
                 <select

plugins/airtable/src/SelectDataSource.tsx

@@ -74,6 +74,9 @@ export function SelectDataSource({ collection, onSelectDataSource }: SelectDataS
                     setSelectedTableId(tables[0]?.id ?? "")
                 } catch (error) {
                     console.error(error)
+
+                    if (abortController.signal.aborted) return
+
                     setStatus("error-tables")
 
                     const baseName = selectedBase?.name ?? selectedBaseId

plugins/airtable/src/data.ts

@@ -28,6 +28,7 @@ export const PLUGIN_KEYS = {
     TABLE_ID: "airtablePluginTableId",
     TABLE_NAME: "airtablePluginTableName",
     SLUG_FIELD_ID: "airtablePluginSlugId",
+    LAST_SYNCED: "airtablePluginLastSynced",
 } as const
 
 const IMAGE_FILE_MIME_TYPES = ["image/jpeg", "image/png", "image/gif", "image/apng", "image/webp", "image/svg+xml"]
@@ -310,12 +311,46 @@ export function getFieldDataEntryForFieldSchema(
     }
 }
 
-export async function getItems(dataSource: DataSource, slugFieldId: string) {
-    const items = await fetchRecords(dataSource.baseId, dataSource.tableId)
+/**
+ * Returns true if the field is a last modified time field that tracks all fields
+ * in the base (i.e. referencedFieldIds is null or empty). Fields that only track
+ * specific columns are not reliable enough to use for sync optimization.
+ */
+export function isFullLastModifiedTimeField(field: PossibleField): boolean {
+    if (field.airtableType !== "lastModifiedTime") return false
+    const options = field.airtableOptions as { isValid?: boolean; referencedFieldIds?: string[] | null } | undefined
+    const referencedFieldIds = options?.referencedFieldIds
+    return options?.isValid === true && (referencedFieldIds == null || referencedFieldIds.length === 0)
+}
+
+export function isUnchangedSinceLastSync(lastModifiedTime: string, lastSyncedTime: string | null): boolean {
+    if (!lastSyncedTime) return false
+
+    const lastModified = new Date(lastModifiedTime)
+    const lastSynced = new Date(lastSyncedTime)
+
+    return lastSynced > lastModified
+}
+
+export async function getItems(dataSource: DataSource, slugFieldId: string, lastSyncedTime: string | null) {
+    const records = await fetchRecords(dataSource.baseId, dataSource.tableId)
     const fieldsById = new Map(dataSource.fields.map(field => [field.id, field]))
+
+    const lastModifiedTimeFieldId = dataSource.fields.find(isFullLastModifiedTimeField)?.id ?? null
+
+    const allRecordIds: string[] = []
     const itemsData: { id: string; slugValue: string; fieldData: FieldDataInput }[] = []
 
-    for (const item of items) {
+    for (const item of records) {
+        allRecordIds.push(item.id)
+
+        if (lastSyncedTime && lastModifiedTimeFieldId) {
+            const lastModifiedValue = item.fields[lastModifiedTimeFieldId]
+            if (typeof lastModifiedValue === "string" && isUnchangedSinceLastSync(lastModifiedValue, lastSyncedTime)) {
+                continue
+            }
+        }
+
         const fieldData: FieldDataInput = {}
 
         for (const fieldSchema of dataSource.fields) {
@@ -417,7 +452,7 @@ export async function getItems(dataSource: DataSource, slugFieldId: string) {
         itemsData.push({ id: item.id, slugValue: slugify(slugField.value as string), fieldData })
     }
 
-    return itemsData
+    return { items: itemsData, allRecordIds }
 }
 
 export interface DataSource {
@@ -469,21 +504,30 @@ export async function syncCollection(
     collection: ManagedCollection,
     dataSource: DataSource,
     fields: readonly PossibleField[],
-    slugFieldId: string
+    slugFieldId: string,
+    previousLastSynced: string | null,
+    syncStartedAtDate: string
 ) {
-    const dataSourceItems = await getItems({ ...dataSource, fields }, slugFieldId)
+    const { items: dataSourceItems, allRecordIds } = await getItems(
+        { ...dataSource, fields },
+        slugFieldId,
+        previousLastSynced
+    )
     const items: ManagedCollectionItemInput[] = []
     const unsyncedItems = new Set(await collection.getItemIds())
 
+    // Remove all fetched record IDs from unsyncedItems so unchanged items are not deleted
+    for (const recordId of allRecordIds) {
+        unsyncedItems.delete(recordId)
+    }
+
     for (const item of dataSourceItems) {
         items.push({
             id: item.id,
             slug: item.slugValue,
             draft: false,
             fieldData: item.fieldData,
         })
-
-        unsyncedItems.delete(item.id)
     }
 
     // Find duplicate slugs and report error if any are found
@@ -504,20 +548,28 @@ export async function syncCollection(
         throw new Error(`Duplicate slug${pluralSuffix} found: ${slugList}. Each item must have a unique slug.`)
     }
 
-    await collection.removeItems(Array.from(unsyncedItems))
-    await collection.addItems(items)
+    const itemsToRemove = Array.from(unsyncedItems)
+    if (itemsToRemove.length > 0) {
+        await collection.removeItems(itemsToRemove)
+    }
+    if (items.length > 0) {
+        await collection.addItems(items)
+    }
+    await collection.setItemOrder(allRecordIds)
 
     await Promise.all([
         collection.setPluginData(PLUGIN_KEYS.BASE_ID, dataSource.baseId),
         collection.setPluginData(PLUGIN_KEYS.TABLE_ID, dataSource.tableId),
         collection.setPluginData(PLUGIN_KEYS.TABLE_NAME, dataSource.tableName),
         collection.setPluginData(PLUGIN_KEYS.SLUG_FIELD_ID, slugFieldId),
+        collection.setPluginData(PLUGIN_KEYS.LAST_SYNCED, syncStartedAtDate),
     ])
 }
 
 export const syncMethods = [
     "ManagedCollection.removeItems",
     "ManagedCollection.addItems",
+    "ManagedCollection.setItemOrder",
     "ManagedCollection.setPluginData",
 ] as const satisfies ProtectedMethod[]
 
@@ -526,7 +578,8 @@ export async function syncExistingCollection(
     previousBaseId: string | null,
     previousTableId: string | null,
     previousTableName: string | null,
-    previousSlugFieldId: string | null
+    previousSlugFieldId: string | null,
+    previousLastSynced: string | null
 ): Promise<{ didSync: boolean }> {
     if (!previousBaseId || !previousTableId) {
         return { didSync: false }
@@ -541,7 +594,9 @@ export async function syncExistingCollection(
     }
 
     try {
-        await framer.hideUI()
+        void framer.hideUI()
+
+        const syncStartedAtDate = new Date().toISOString()
 
         const existingFields = await collection.getFields()
         const table = await fetchTable(previousBaseId, previousTableId)
@@ -574,7 +629,14 @@ export async function syncExistingCollection(
             tableName: table.name,
             fields: mergedFields,
         }
-        await syncCollection(collection, dataSource, fieldsToSync, previousSlugFieldId)
+        await syncCollection(
+            collection,
+            dataSource,
+            fieldsToSync,
+            previousSlugFieldId,
+            previousLastSynced,
+            syncStartedAtDate
+        )
         return { didSync: true }
     } catch (error) {
         console.error(error)

plugins/airtable/src/main.tsx

@@ -33,19 +33,23 @@ if (!tokens) {
     })
 }
 
-const [previousBaseId, previousTableId, previousTableName, previousSlugFieldId] = await Promise.all([
-    activeCollection.getPluginData(PLUGIN_KEYS.BASE_ID),
-    activeCollection.getPluginData(PLUGIN_KEYS.TABLE_ID),
-    activeCollection.getPluginData(PLUGIN_KEYS.TABLE_NAME),
-    activeCollection.getPluginData(PLUGIN_KEYS.SLUG_FIELD_ID),
-])
+const [previousBaseId, previousTableId, previousTableName, previousSlugFieldId, previousLastSynced] = await Promise.all(
+    [
+        activeCollection.getPluginData(PLUGIN_KEYS.BASE_ID),
+        activeCollection.getPluginData(PLUGIN_KEYS.TABLE_ID),
+        activeCollection.getPluginData(PLUGIN_KEYS.TABLE_NAME),
+        activeCollection.getPluginData(PLUGIN_KEYS.SLUG_FIELD_ID),
+        activeCollection.getPluginData(PLUGIN_KEYS.LAST_SYNCED),
+    ]
+)
 
 const { didSync } = await syncExistingCollection(
     activeCollection,
     previousBaseId,
     previousTableId,
     previousTableName,
-    previousSlugFieldId
+    previousSlugFieldId,
+    previousLastSynced
 )
 
 if (didSync) {
@@ -60,6 +64,7 @@ if (didSync) {
                 previousBaseId={previousBaseId}
                 previousTableId={previousTableId}
                 previousSlugFieldId={previousSlugFieldId}
+                previousLastSynced={previousLastSynced}
             />
         </React.StrictMode>
     )