From 76aa6c2d249f6cfa6e33cbd01861f2a365d55377 Mon Sep 17 00:00:00 2001 From: Dhika Rizky Date: Tue, 15 Oct 2024 22:43:56 +0700 Subject: [PATCH 1/5] feat: add idb driver --- src/utils/indexeddb.ts | 129 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 src/utils/indexeddb.ts diff --git a/src/utils/indexeddb.ts b/src/utils/indexeddb.ts new file mode 100644 index 0000000..4753a45 --- /dev/null +++ b/src/utils/indexeddb.ts @@ -0,0 +1,129 @@ +import { RouteRawData, ShapeRawData, StopRawData, TripRawData } from "../data/consumer"; + +const DB_VERSION = 1; +const DB_NAME = "opentije-data"; + +export async function openIDB(): Promise { + return new Promise((resolve, reject) => { + const openResponse = indexedDB.open(DB_NAME, DB_VERSION); + + openResponse.onerror = () => { + reject(openResponse.error); + }; + + openResponse.onsuccess = () => { + resolve(openResponse.result); + }; + + openResponse.onupgradeneeded = (event) => { + const db = openResponse.result; + if (event.oldVersion === 0) { + db.createObjectStore("routes", { keyPath: "route_id" }); + db.createObjectStore("stops", { keyPath: "stop_id" }); + db.createObjectStore("shapes", { keyPath: "shape_id" }); + db.createObjectStore("trips", { keyPath: "trip_id" }); + // metadata only need key & value + db.createObjectStore("meta", { keyPath: "key" }); + } + }; + }); +} + +export async function getAll( + db: IDBDatabase, + storeName: "routes" | "stops" | "shapes" | "trips" | "meta" +): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(storeName, "readonly"); + const store = tx.objectStore(storeName); + const query = store.getAll(); + tx.commit(); + + query.onsuccess = () => { + const result = query.result as T[]; + resolve(result); + } + + query.onerror = (event) => { + reject(query.error); + } + }); +} + +export async function getGTFSCacheFromIDB(db: IDBDatabase) { + const getRoutes = async (): Promise => { + return await getAll(db, "routes"); + } + + const getStops = async (): Promise => { + return await getAll(db, "stops"); + } + + const getShapes = async (): Promise => { + return await getAll(db, "shapes"); + } + + const getTrips = async (): Promise => { + return await getAll(db, "trips"); + } + + const resetMeta = async (): Promise => { + const tx = db.transaction("meta", "readwrite"); + const store = tx.objectStore("meta"); + store.clear(); + tx.commit(); + } + + const hydrateFromParsedGTFS = async ( + routeRawDatum: RouteRawData[], + stopRawDatum: StopRawData[], + shapeRawDatum: ShapeRawData[], + tripRawDatum: TripRawData[] + ): Promise => { + return new Promise((resolve, reject) => { + const tx = db.transaction([ + "routes", + "stops", + "shapes", + "trips", + "meta" + ]); + + const routesStore = tx.objectStore("routes"); + for (const routeRawData of routeRawDatum) { + routesStore.put(routeRawData); + } + + const stopsStore = tx.objectStore("stops"); + for (const stopRawData of stopRawDatum) { + stopsStore.put(stopRawData); + } + + const shapesStore = tx.objectStore("shapes"); + for (const shapeRawData of shapeRawDatum) { + shapesStore.put(shapeRawData); + } + + const tripsStore = tx.objectStore("trips"); + for (const tripRawData of tripRawDatum) { + tripsStore.put(tripRawData); + } + + const metaStore = tx.objectStore("meta"); + metaStore.put({ key: "expireAt", value: new Date().getTime() }); + + tx.commit(); + + tx.oncomplete = () => resolve(); + tx.onerror = () => reject(tx.error); + }); + } + + return { + getRoutes, + getStops, + getShapes, + getTrips, + hydrateFromParsedGTFS + } +} From 8419637ad6d8ef2fc98901b45e001892e36e191e Mon Sep 17 00:00:00 2001 From: Dhika Rizky Date: Tue, 15 Oct 2024 22:53:18 +0700 Subject: [PATCH 2/5] fix: skip invalid data on hydration --- src/utils/indexeddb.ts | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/utils/indexeddb.ts b/src/utils/indexeddb.ts index 4753a45..6f97190 100644 --- a/src/utils/indexeddb.ts +++ b/src/utils/indexeddb.ts @@ -87,26 +87,46 @@ export async function getGTFSCacheFromIDB(db: IDBDatabase) { "shapes", "trips", "meta" - ]); + ], "readwrite"); const routesStore = tx.objectStore("routes"); for (const routeRawData of routeRawDatum) { - routesStore.put(routeRawData); + try { + routesStore.put(routeRawData); + } catch (e) { + // skip on invalid data + continue; + } } const stopsStore = tx.objectStore("stops"); for (const stopRawData of stopRawDatum) { - stopsStore.put(stopRawData); + try { + stopsStore.put(stopRawData); + } catch (e) { + // skip on invalid data + continue; + } } const shapesStore = tx.objectStore("shapes"); for (const shapeRawData of shapeRawDatum) { - shapesStore.put(shapeRawData); + try { + shapesStore.put(shapeRawData); + } catch (e) { + // skip on invalid data + continue; + } } const tripsStore = tx.objectStore("trips"); for (const tripRawData of tripRawDatum) { - tripsStore.put(tripRawData); + try { + tripsStore.put(tripRawData); + } catch (e) { + // skip on invalid data + continue; + } } const metaStore = tx.objectStore("meta"); From ff8e0b8bb31a38d7efca6f486b73af74dc087ddb Mon Sep 17 00:00:00 2001 From: Dhika Rizky Date: Tue, 15 Oct 2024 23:06:54 +0700 Subject: [PATCH 3/5] feat: add simple caching with idb --- src/data/consumer.ts | 31 +++++++++++++++++++++++++++++++ src/utils/indexeddb.ts | 8 ++++++++ 2 files changed, 39 insertions(+) diff --git a/src/data/consumer.ts b/src/data/consumer.ts index aea6207..ef0c6f9 100644 --- a/src/data/consumer.ts +++ b/src/data/consumer.ts @@ -1,5 +1,6 @@ import { BlobReader, Entry, TextWriter, ZipReader } from "@zip.js/zip.js"; import Papa from "papaparse"; +import { openIDB, getGTFSCacheFromIDB } from "../utils/indexeddb"; export enum RouteType { BRT = "BRT", @@ -12,6 +13,23 @@ export enum RouteType { } export async function getRawData() { + try { + const idb = await openIDB(); + const cache = await getGTFSCacheFromIDB(idb); + const isCacheStale = await cache.checkIfStale(); + if (!isCacheStale) { + return { + routesRawData: await cache.getRoutes(), + stopsRawData: await cache.getStops(), + tripsRawData: await cache.getTrips(), + shapesRawData: await cache.getShapes(), + stopTimesRawData: [] // TODO + } + } + } catch (e) { + // do nothing, keep parsing + } + const response = await fetch("/assets/file_gtfs.zip"); const blob = await response.blob(); const reader = new BlobReader(blob); @@ -89,6 +107,19 @@ export async function getRawData() { await zipReader.close(); + try { + const idb = await openIDB(); + const cache = await getGTFSCacheFromIDB(idb); + await cache.hydrateFromParsedGTFS( + routesRawData, + stopsRawData, + shapesRawData, + tripsRawData + ); + } catch (e) { + // do nothing + } + return { routesRawData, stopsRawData, diff --git a/src/utils/indexeddb.ts b/src/utils/indexeddb.ts index 6f97190..943af75 100644 --- a/src/utils/indexeddb.ts +++ b/src/utils/indexeddb.ts @@ -74,6 +74,13 @@ export async function getGTFSCacheFromIDB(db: IDBDatabase) { tx.commit(); } + const checkIfStale = async (): Promise => { + const meta = await getAll<{ key: string, value: string | number }>(db, "meta"); + const expireAt = meta.find((item) => item.key === "expireAt")?.value; + if (expireAt === undefined) return false; + return (expireAt as number) > new Date().getTime(); + } + const hydrateFromParsedGTFS = async ( routeRawDatum: RouteRawData[], stopRawDatum: StopRawData[], @@ -144,6 +151,7 @@ export async function getGTFSCacheFromIDB(db: IDBDatabase) { getStops, getShapes, getTrips, + checkIfStale, hydrateFromParsedGTFS } } From 6ce8f0013ba25a5a83deda48e110e282dd4f6ff3 Mon Sep 17 00:00:00 2001 From: Dhika Rizky Date: Tue, 15 Oct 2024 23:13:33 +0700 Subject: [PATCH 4/5] fix: wrong stale logic --- src/utils/indexeddb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/indexeddb.ts b/src/utils/indexeddb.ts index 943af75..0888528 100644 --- a/src/utils/indexeddb.ts +++ b/src/utils/indexeddb.ts @@ -77,7 +77,7 @@ export async function getGTFSCacheFromIDB(db: IDBDatabase) { const checkIfStale = async (): Promise => { const meta = await getAll<{ key: string, value: string | number }>(db, "meta"); const expireAt = meta.find((item) => item.key === "expireAt")?.value; - if (expireAt === undefined) return false; + if (expireAt === undefined) return true; return (expireAt as number) > new Date().getTime(); } From 1446726b039c70876103928d049c50da3437336f Mon Sep 17 00:00:00 2001 From: Dhika Rizky Date: Tue, 15 Oct 2024 23:14:53 +0700 Subject: [PATCH 5/5] feat: do caching in background --- src/data/consumer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/data/consumer.ts b/src/data/consumer.ts index ef0c6f9..d6b97d3 100644 --- a/src/data/consumer.ts +++ b/src/data/consumer.ts @@ -110,7 +110,7 @@ export async function getRawData() { try { const idb = await openIDB(); const cache = await getGTFSCacheFromIDB(idb); - await cache.hydrateFromParsedGTFS( + cache.hydrateFromParsedGTFS( routesRawData, stopsRawData, shapesRawData,