Mirabuf Caching

fission/package.json

@@ -26,6 +26,7 @@
     "@mui/system": "^5.15.20",
     "@react-three/drei": "^9.96.5",
     "@react-three/fiber": "^8.15.15",
+    "@types/crypto-js": "^4.2.2",
     "@vitest/browser": "^1.6.0",
     "@vitest/coverage-v8": "^1.6.0",
     "async-mutex": "^0.5.0",
@@ -55,6 +56,7 @@
     "@vitejs/plugin-react-swc": "^3.5.0",
     "autoprefixer": "^10.4.14",
     "cssnano": "^6.0.1",
+    "crypto-js": "^4.2.0",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^8.8.0",
     "eslint-import-resolver-alias": "^1.1.2",
@@ -68,7 +70,7 @@
     "protobufjs-cli": "^1.1.2",
     "tailwindcss": "^3.3.3",
     "tsconfig-paths": "^4.2.0",
-    "typescript": "^5.2.2",
+    "typescript": "^5.4.5",
     "vite": "^5.1.4",
     "vite-plugin-singlefile": "^0.13.5",
     "vitest": "^1.5.3"

fission/src/Synthesis.tsx

@@ -1,6 +1,6 @@
 import Scene from "@/components/Scene.tsx"
 import MirabufSceneObject from "./mirabuf/MirabufSceneObject.ts"
-import { LoadMirabufRemote } from "./mirabuf/MirabufLoader.ts"
+import { LoadMirabufRemote, MiraType } from "./mirabuf/MirabufLoader.ts"
 import { mirabuf } from "./proto/mirabuf"
 import MirabufParser, { ParseErrorSeverity } from "./mirabuf/MirabufParser.ts"
 import MirabufInstance from "./mirabuf/MirabufInstance.ts"
@@ -92,8 +92,8 @@ function Synthesis() {
         console.log(urlParams)
 
         const setup = async () => {
-            const miraAssembly = await LoadMirabufRemote(mira_path)
-                .catch(_ => LoadMirabufRemote(DEFAULT_MIRA_PATH))
+            const miraAssembly = await LoadMirabufRemote(mira_path, MiraType.ROBOT)
+                .catch(_ => LoadMirabufRemote(DEFAULT_MIRA_PATH, MiraType.ROBOT))
                 .catch(console.error)
 
             await (async () => {

fission/src/mirabuf/MirabufLoader.ts

@@ -1,26 +1,130 @@
-import { mirabuf } from "../proto/mirabuf"
+import { mirabuf } from "@/proto/mirabuf"
 import Pako from "pako"
-// import * as fs from "fs"
+
+type MiraCache = { [id: string] : string}
+
+const robotsDirName = "Robots"
+const fieldsDirName = "Fields"
+const root = await navigator.storage.getDirectory()
+const robotFolderHandle = await root.getDirectoryHandle(robotsDirName, { create: true })
+const fieldFolderHandle = await root.getDirectoryHandle(fieldsDirName, { create: true })
 
 export function UnzipMira(buff: Uint8Array): Uint8Array {
+    // Check if file is gzipped via magic gzip numbers 31 139
     if (buff[0] == 31 && buff[1] == 139) {
         return Pako.ungzip(buff)
     } else {
         return buff
     }
 }
 
-export async function LoadMirabufRemote(
-    fetchLocation: string,
-    useCache: boolean = true
-): Promise<mirabuf.Assembly | undefined> {
-    const miraBuff = await fetch(encodeURI(fetchLocation), useCache ? undefined : { cache: "no-store" })
-        .then(x => x.blob())
-        .then(x => x.arrayBuffer())
-    const byteBuffer = UnzipMira(new Uint8Array(miraBuff))
-    return mirabuf.Assembly.decode(byteBuffer)
+export async function LoadMirabufRemote(fetchLocation: string, type: MiraType, blobHashID?: string): Promise<mirabuf.Assembly | undefined> {
+    const map = GetMiraCacheMap(type)
+
+    // blobHashID for ImportLocalMira
+    const targetID = map ? ( blobHashID ? map[blobHashID] : map[fetchLocation]) : undefined
+
+    if (targetID != undefined && map != undefined) {
+        console.log("Loading mira from cache")
+        return (await LoadMirabufCache(fetchLocation, targetID, type, map)) ?? LoadAndCacheMira(fetchLocation, type, blobHashID)
+    } else {
+        console.log("Loading and caching new mira")
+        return await LoadAndCacheMira(fetchLocation, type, blobHashID)
+    }
 }
 
 // export function LoadMirabufLocal(fileLocation: string): mirabuf.Assembly {
 //     return mirabuf.Assembly.decode(UnzipMira(new Uint8Array(fs.readFileSync(fileLocation))))
 // }
+
+export async function ClearMira() {
+    for await (const key of robotFolderHandle.keys()) {
+        robotFolderHandle.removeEntry(key)
+    }
+    for await (const key of fieldFolderHandle.keys()) {
+        fieldFolderHandle.removeEntry(key)
+    }
+
+    window.localStorage.removeItem(robotsDirName)
+    window.localStorage.removeItem(fieldsDirName)
+}
+
+export function GetMiraCacheMap(type: MiraType): MiraCache | undefined {
+    const miraJSON = window.localStorage.getItem(type == MiraType.ROBOT ? robotsDirName : fieldsDirName)
+
+    if (miraJSON != undefined) {
+        console.log("mirabuf JSON found")
+        console.log(miraJSON)
+        return JSON.parse(miraJSON)
+    } else {
+        console.log("mirabuf JSON not found")
+        return undefined
+    }
+}
+
+async function LoadAndCacheMira(fetchLocation: string, type: MiraType, blobHashID?: string): Promise<mirabuf.Assembly | undefined> {
+    try {
+        // Grab file remote
+        const miraBuff = await fetch(encodeURI(fetchLocation), import.meta.env.DEV ? { cache: "no-store" } : undefined)
+            .then(x => x.blob())
+            .then(x => x.arrayBuffer())
+        const byteBuffer = UnzipMira(new Uint8Array(miraBuff))
+        const assembly = mirabuf.Assembly.decode(byteBuffer)
+
+        // Store in OPFS
+        const backupID = Date.now().toString()
+        const fileHandle = await (type == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle(
+            backupID,
+            { create: true }
+        )
+        const writable = await fileHandle.createWritable()
+        await writable.write(miraBuff)
+        await writable.close()
+
+        // Local cache map
+        const map: MiraCache = GetMiraCacheMap(type) ?? {}
+        map[blobHashID ?? fetchLocation] = backupID
+        window.localStorage.setItem(type == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map))
+
+        console.log(assembly)
+        return assembly
+    } catch (e) {
+        console.error("Failed to load and cache mira " + e)
+        return undefined
+    }
+}
+
+async function LoadMirabufCache(
+    fetchLocation: string,
+    targetID: string,
+    type: MiraType,
+    map: MiraCache
+): Promise<mirabuf.Assembly | undefined> {
+    try {
+        const fileHandle =
+            (await (type == MiraType.ROBOT ? robotFolderHandle : fieldFolderHandle).getFileHandle(targetID, {
+                create: false,
+            })) ?? undefined
+
+        // Get assembly from file
+        if (fileHandle != undefined) {
+            const buff = await fileHandle.getFile().then(x => x.arrayBuffer())
+            const assembly = mirabuf.Assembly.decode(UnzipMira(new Uint8Array(buff)))
+            console.log(assembly)
+            return assembly
+        }
+    } catch (e) {
+        console.error("Failed to find file from OPFS" + e)
+
+        // Remove from localStorage list
+        delete map[fetchLocation]
+        window.localStorage.setItem(type == MiraType.ROBOT ? robotsDirName : fieldsDirName, JSON.stringify(map))
+
+        return undefined
+    }
+}
+
+export enum MiraType {
+    ROBOT,
+    FIELD,
+}

fission/src/mirabuf/MirabufParser.ts

@@ -1,6 +1,6 @@
 import * as THREE from "three"
-import { mirabuf } from "../proto/mirabuf"
-import { MirabufTransform_ThreeMatrix4 } from "../util/TypeConversions"
+import { mirabuf } from "@/proto/mirabuf"
+import { MirabufTransform_ThreeMatrix4 } from "@/util/TypeConversions"
 
 export enum ParseErrorSeverity {
     Unimportable = 10,

fission/src/mirabuf/MirabufSceneObject.ts

@@ -1,7 +1,7 @@
 import { mirabuf } from "@/proto/mirabuf"
 import SceneObject from "../systems/scene/SceneObject"
 import MirabufInstance from "./MirabufInstance"
-import { LoadMirabufRemote } from "./MirabufLoader"
+import { LoadMirabufRemote, MiraType } from "./MirabufLoader"
 import MirabufParser, { ParseErrorSeverity } from "./MirabufParser"
 import World from "@/systems/World"
 import Jolt from "@barclah/jolt-physics"
@@ -157,8 +157,12 @@ class MirabufSceneObject extends SceneObject {
     }
 }
 
-export async function CreateMirabufFromUrl(path: string): Promise<MirabufSceneObject | null | undefined> {
-    const miraAssembly = await LoadMirabufRemote(path).catch(console.error)
+export async function CreateMirabufFromUrl(
+    path: string,
+    miraType: MiraType,
+    hashID?: string
+): Promise<MirabufSceneObject | null | undefined> {
+    const miraAssembly = await LoadMirabufRemote(path, miraType, hashID).catch(console.error)
 
     if (!miraAssembly || !(miraAssembly instanceof mirabuf.Assembly)) {
         return