IndexedDB Caching

src/data/consumer.ts

@@ -1,5 +1,6 @@
 import { BlobReader, Entry, TextWriter, ZipReader } from "@zip.js/zip.js";
 import Papa from "papaparse";
+import { openIDB, getGTFSCacheFromIDB } from "../utils/indexeddb";
 
 export enum RouteType {
     BRT = "BRT",
@@ -12,6 +13,23 @@ export enum RouteType {
 }
 
 export async function getRawData() {
+    try {
+        const idb = await openIDB();
+        const cache = await getGTFSCacheFromIDB(idb);
+        const isCacheStale = await cache.checkIfStale();
+        if (!isCacheStale) {
+            return {
+                routesRawData: await cache.getRoutes(),
+                stopsRawData: await cache.getStops(),
+                tripsRawData: await cache.getTrips(),
+                shapesRawData: await cache.getShapes(),
+                stopTimesRawData: [] // TODO
+            }
+        }
+    } catch (e) {
+        // do nothing, keep parsing
+    }
+
     const response = await fetch("/assets/file_gtfs.zip");
     const blob = await response.blob();
     const reader = new BlobReader(blob);
@@ -89,6 +107,19 @@ export async function getRawData() {
 
     await zipReader.close();
 
+    try {
+        const idb = await openIDB();
+        const cache = await getGTFSCacheFromIDB(idb);
+        cache.hydrateFromParsedGTFS(
+            routesRawData,
+            stopsRawData,
+            shapesRawData,
+            tripsRawData
+        );
+    } catch (e) {
+        // do nothing
+    }
+
     return {
         routesRawData,
         stopsRawData,

src/utils/indexeddb.ts

@@ -0,0 +1,157 @@
+import { RouteRawData, ShapeRawData, StopRawData, TripRawData } from "../data/consumer";
+
+const DB_VERSION = 1;
+const DB_NAME = "opentije-data";
+
+export async function openIDB(): Promise<IDBDatabase> {
+    return new Promise((resolve, reject) => {
+        const openResponse = indexedDB.open(DB_NAME, DB_VERSION);
+
+        openResponse.onerror = () => {
+            reject(openResponse.error);
+        };
+
+        openResponse.onsuccess = () => {
+            resolve(openResponse.result);
+        };
+
+        openResponse.onupgradeneeded = (event) => {
+            const db = openResponse.result;
+            if (event.oldVersion === 0) {
+                db.createObjectStore("routes", { keyPath: "route_id" });
+                db.createObjectStore("stops", { keyPath: "stop_id" });
+                db.createObjectStore("shapes", { keyPath: "shape_id" });
+                db.createObjectStore("trips", { keyPath: "trip_id" });
+                // metadata only need key & value
+                db.createObjectStore("meta", { keyPath: "key" });
+            }
+        };
+    });
+}
+
+export async function getAll<T = unknown>(
+    db: IDBDatabase,
+    storeName: "routes" | "stops" | "shapes" | "trips" | "meta"
+): Promise<T[]> {
+    return new Promise((resolve, reject) => {
+        const tx = db.transaction(storeName, "readonly");
+        const store = tx.objectStore(storeName);
+        const query = store.getAll();
+        tx.commit();
+
+        query.onsuccess = () => {
+            const result = query.result as T[];
+            resolve(result);
+        }
+
+        query.onerror = (event) => {
+            reject(query.error);
+        }
+    });
+}
+
+export async function getGTFSCacheFromIDB(db: IDBDatabase) {
+    const getRoutes = async (): Promise<RouteRawData[]> => {
+        return await getAll<RouteRawData>(db, "routes");
+    }
+
+    const getStops = async (): Promise<StopRawData[]> => {
+        return await getAll<StopRawData>(db, "stops");
+    }
+
+    const getShapes = async (): Promise<ShapeRawData[]> => {
+        return await getAll<ShapeRawData>(db, "shapes");
+    }
+
+    const getTrips = async (): Promise<TripRawData[]> => {
+        return await getAll<TripRawData>(db, "trips");
+    }
+
+    const resetMeta = async (): Promise<void> => {
+        const tx = db.transaction("meta", "readwrite");
+        const store = tx.objectStore("meta");
+        store.clear();
+        tx.commit();
+    }
+
+    const checkIfStale = async (): Promise<boolean> => {
+        const meta = await getAll<{ key: string, value: string | number }>(db, "meta");
+        const expireAt = meta.find((item) => item.key === "expireAt")?.value;
+        if (expireAt === undefined) return true;
+        return (expireAt as number) > new Date().getTime();
+    }
+
+    const hydrateFromParsedGTFS = async (
+        routeRawDatum: RouteRawData[],
+        stopRawDatum: StopRawData[],
+        shapeRawDatum: ShapeRawData[],
+        tripRawDatum: TripRawData[]
+    ): Promise<void> => {
+        return new Promise((resolve, reject) => {
+            const tx = db.transaction([
+                "routes",
+                "stops",
+                "shapes",
+                "trips",
+                "meta"
+            ], "readwrite");
+
+            const routesStore = tx.objectStore("routes");
+            for (const routeRawData of routeRawDatum) {
+                try {
+                    routesStore.put(routeRawData);
+                } catch (e) {
+                    // skip on invalid data
+                    continue;
+                }
+            }
+
+            const stopsStore = tx.objectStore("stops");
+            for (const stopRawData of stopRawDatum) {
+                try {
+                    stopsStore.put(stopRawData);
+                } catch (e) {
+                    // skip on invalid data
+                    continue;
+                }
+            }
+
+            const shapesStore = tx.objectStore("shapes");
+            for (const shapeRawData of shapeRawDatum) {
+                try {
+                    shapesStore.put(shapeRawData);
+                } catch (e) {
+                    // skip on invalid data
+                    continue;
+                }
+            }
+
+            const tripsStore = tx.objectStore("trips");
+            for (const tripRawData of tripRawDatum) {
+                try {
+                    tripsStore.put(tripRawData);
+                } catch (e) {
+                    // skip on invalid data
+                    continue;
+                }
+            }
+
+            const metaStore = tx.objectStore("meta");
+            metaStore.put({ key: "expireAt", value: new Date().getTime() });
+
+            tx.commit();
+
+            tx.oncomplete = () => resolve();
+            tx.onerror = () => reject(tx.error);
+        });
+    }
+
+    return {
+        getRoutes,
+        getStops,
+        getShapes,
+        getTrips,
+        checkIfStale,
+        hydrateFromParsedGTFS
+    }
+}