feat: add an app for comfyUI to run in captain (#269)

## Motivation

- adds a guarded window
- adds a guarded preload script that could be filled with safe IPC calls
- uses webview top display comfyUI
- loads comfy under given port
- unloads models and frees memory when comfywindow is closed

> [!NOTE]
> if the queue is still open from another client, this might cause
issues (lost queue or job).
> We need to discuss this

nextron.config.js

@@ -14,6 +14,7 @@ module.exports = {
 			background: "./src/electron/background.ts",
 			preload: "./src/electron/preload.ts",
 			"app-preload": "./src/electron/app-preload.ts",
+			"guarded-preload": "./src/electron/guarded-preload.ts",
 		};
 		return config;
 	},

package.json

@@ -52,10 +52,10 @@
     "sharp": "0.33.2"
   },
   "devDependencies": {
-    "@captn/joy": "^0.28.1",
-    "@captn/react": "^0.28.0",
-    "@captn/theme": "^0.28.0",
-    "@captn/utils": "^0.28.0",
+    "@captn/joy": "^0.29.0",
+    "@captn/react": "^0.29.0",
+    "@captn/theme": "^0.29.0",
+    "@captn/utils": "^0.29.0",
     "@commitlint/cli": "^19.3.0",
     "@commitlint/config-conventional": "^19.2.2",
     "@dnd-kit/core": "^6.1.0",

resources/actions/comfy-ui/readme.md

@@ -0,0 +1,41 @@
+---
+id: comfy-ui
+label: ComfyUI
+language: en
+fileId: comfy-ui
+type: app
+creatorID: Blibla
+tags:
+  - comfyUI
+  - stable diffusion
+  - img2img
+  - txt2img
+  - apps
+license: AGPL 3.0
+accessLevel: public
+description: "The most powerful and modular stable diffusion GUI and backend"
+icon: ResistorNodeIcon
+iconColor: "#353535"
+---
+
+Open the app comfyUI.
+I want to generate images with stable diffusion.
+Accelerate creative workflows with ComfyUI’s modular stable diffusion GUI.
+Unleash image synthesis power using ComfyUI for txt2img and img2img tasks.
+Navigate advanced node-based interfaces with ease in ComfyUI.
+Optimize image generation with ComfyUI's intuitive stable diffusion tools.
+Harness the full potential of stable diffusion models with ComfyUI.
+Transform digital art processes with ComfyUI’s robust features.
+Empower your image projects with ComfyUI’s dynamic GUI.
+Enhance productivity using ComfyUI's streamlined image creation system.
+Customize stable diffusion operations efficiently with ComfyUI.
+Achieve superior image results through ComfyUI’s innovative technology.
+Deploy ComfyUI for quick, effective stable diffusion model setups.
+Leverage ComfyUI’s flexible architecture for diverse imaging needs.
+Maximize creative output with ComfyUI’s powerful imaging solutions.
+Simplify complex workflows in image processing with ComfyUI.
+Engage ComfyUI for high-performance image generation.
+Utilize ComfyUI for detailed and artistic stable diffusion applications.
+versatile image synthesis options within ComfyUI.
+stable diffusion pipeline using ComfyUI’s smart interface.
+advanced imaging techniques effortlessly with ComfyUI.

src/client/atoms/icons/dynamic.tsx

@@ -2,8 +2,7 @@ import { localFile } from "@captn/utils/string";
 import Box from "@mui/joy/Box";
 import type { SxProps } from "@mui/joy/styles/types";
 import type { DefaultComponentProps } from "@mui/material/OverridableComponent";
-import type { SvgIconProps } from "@mui/material/SvgIcon";
-import type { SvgIconTypeMap } from "@mui/material/SvgIcon";
+import type { SvgIconProps, SvgIconTypeMap } from "@mui/material/SvgIcon";
 import { useAtom } from "jotai/index";
 import dynamic from "next/dynamic";
 import type { ComponentType } from "react";
@@ -25,6 +24,7 @@ const Settings = dynamic(() => import("@mui/icons-material/Settings"));
 const ShoppingBag = dynamic(() => import("@mui/icons-material/ShoppingBag"));
 const Stream = dynamic(() => import("@mui/icons-material/Stream"));
 const QuestionMark = dynamic(() => import("@mui/icons-material/QuestionMark"));
+const ResistorNodeIcon = dynamic(() => import("./resistor-node-icon"));
 
 const iconCache: Record<
 	string,
@@ -40,6 +40,7 @@ const iconCache: Record<
 	LightMode,
 	MenuBook,
 	QuestionMark,
+	ResistorNodeIcon,
 	Settings,
 	ShoppingBag,
 	Stream,

src/client/atoms/icons/resistor-node-icon.tsx

@@ -0,0 +1,13 @@
+import type { SvgIconProps } from "@mui/material/SvgIcon";
+import SvgIcon from "@mui/material/SvgIcon";
+
+export default function ResistorNodeIcon(properties: SvgIconProps) {
+	return (
+		<SvgIcon {...properties} viewBox="0 0 24 24">
+			<path
+				fill="currentColor"
+				d="M2,11H3.67C4.08,9.83 5.19,9 6.5,9A3,3 0 0,1 9.5,12C9.5,12.65 9.29,13.25 8.94,13.74L10.07,15.35L13.11,4L14.61,6.13L16.7,9.11L17.5,9C18.81,9 19.92,9.83 20.33,11H22V13H20.33C19.92,14.17 18.81,15 17.5,15A3,3 0 0,1 14.5,12C14.5,11.35 14.71,10.75 15.06,10.26L13.93,8.65L10.89,20L7.3,14.89C7.05,14.96 6.78,15 6.5,15C5.19,15 4.08,14.17 3.67,13H2V11M17.5,10.5A1.5,1.5 0 0,0 16,12A1.5,1.5 0 0,0 17.5,13.5A1.5,1.5 0 0,0 19,12A1.5,1.5 0 0,0 17.5,10.5M6.5,10.5A1.5,1.5 0 0,0 5,12A1.5,1.5 0 0,0 6.5,13.5A1.5,1.5 0 0,0 8,12A1.5,1.5 0 0,0 6.5,10.5Z"
+			/>
+		</SvgIcon>
+	);
+}

src/client/pages/_app.tsx

@@ -21,6 +21,7 @@ function App({ Component, pageProps }: AppProps) {
 
 	const isPrompt = pathname === "/[locale]/prompt";
 	const isCore = pathname.startsWith("/[locale]/core");
+	const isComfyUI = pathname.startsWith("/comfy-ui");
 	// Intended abuse of useMemo to allow changes on server and client mount
 	useMemo(() => {
 		// We need to set is before the render to tell dayjs to change the locale
@@ -46,7 +47,7 @@ function App({ Component, pageProps }: AppProps) {
 			</Head>
 
 			<CssBaseline />
-			<ActionListeners />
+			{!isComfyUI && <ActionListeners />}
 			{isPrompt && (
 				<Global
 					styles={css({

src/client/pages/comfy-ui.tsx

@@ -0,0 +1,58 @@
+import { useUnload } from "@captn/react/use-unload";
+import Box from "@mui/joy/Box";
+import CircularProgress from "@mui/joy/CircularProgress";
+import Head from "next/head";
+import { useEffect, useState } from "react";
+
+import { buildKey } from "@/shared/build-key";
+import { ID } from "@/shared/enums";
+
+const APP_ID = "comfy-ui";
+
+export default function ComfyUI() {
+	const [port, setPort] = useState("");
+
+	useEffect(() => {
+		window.ipc.send(buildKey([ID.COMFYUI_PORT], { suffix: ":get" }));
+		const unsubscribe = window.ipc.on(
+			buildKey([ID.COMFYUI_PORT], { suffix: ":get" }),
+			(port_: string) => {
+				setPort(port_);
+			}
+		);
+		return () => {
+			unsubscribe();
+		};
+	}, []);
+
+	// Unload models when the window is closed
+	useUnload(APP_ID, "comfyui:free", {
+		unloadModels: true,
+		freeMemory: false,
+	});
+
+	return (
+		<>
+			<Head>
+				<title>ComfyUI</title>
+			</Head>
+			{port ? (
+				<webview
+					src={`http://127.0.0.1:${port}/`}
+					style={{ position: "absolute", inset: 0 }}
+				/>
+			) : (
+				<Box
+					sx={{
+						position: "absolute",
+						top: "50%",
+						left: "50%",
+						transform: "translate(-50%, -50%)",
+					}}
+				>
+					<CircularProgress />
+				</Box>
+			)}
+		</>
+	);
+}

src/electron/guarded-preload.ts

@@ -0,0 +1,34 @@
+import type { SDKMessage } from "@captn/react/types";
+import { APP_MESSAGE_KEY } from "@captn/utils/constants";
+import { contextBridge, ipcRenderer, type IpcRendererEvent } from "electron";
+
+import { buildKey } from "@/shared/build-key";
+import { ID } from "@/shared/enums";
+
+const allowedMethods = new Set([buildKey([ID.COMFYUI_PORT], { suffix: ":get" })]);
+contextBridge.exposeInMainWorld("ipc", {
+	send(channel: string, value?: unknown) {
+		if (allowedMethods.has(channel)) {
+			ipcRenderer.send(channel, value);
+		} else if (
+			channel === APP_MESSAGE_KEY &&
+			(value as { message: SDKMessage<{ freeMemory?: boolean; unloadModels?: boolean }> })
+				?.message?.action === "comfyui:free"
+		) {
+			ipcRenderer.send(channel, value);
+		}
+	},
+	on(channel: string, callback: (...arguments_: unknown[]) => void) {
+		function subscription(_event: IpcRendererEvent, ...arguments_: unknown[]) {
+			return callback(...arguments_);
+		}
+
+		if (allowedMethods.has(channel)) {
+			ipcRenderer.on(channel, subscription);
+
+			return () => {
+				ipcRenderer.removeListener(channel, subscription);
+			};
+		}
+	},
+});

src/electron/helpers/ipc/sdk/comfyui.ts

@@ -3,7 +3,10 @@ import { APP_MESSAGE_KEY } from "@captn/utils/constants";
 import type { IpcMainEvent } from "electron";
 import { ipcMain } from "electron";
 
+import { PORTS } from "@/electron/constants";
 import { ComfyUI } from "@/electron/services/comfyui";
+import { buildKey } from "@/shared/build-key";
+import { ID } from "@/shared/enums";
 import type { NodeChain } from "@/shared/types/comfyui";
 
 ipcMain.on(
@@ -13,6 +16,7 @@ ipcMain.on(
 		{ message, appId }: { message: SDKMessage<T>; appId: string }
 	) => {
 		const channel = `${appId}:${APP_MESSAGE_KEY}`;
+		console.log({ message, appId });
 
 		switch (message.action) {
 			case "comfyui:queue": {
@@ -30,6 +34,27 @@ ipcMain.on(
 				break;
 			}
 
+			case "comfyui:free": {
+				const { unloadModels, freeMemory } = message.payload as {
+					unloadModels?: boolean;
+					freeMemory?: boolean;
+				};
+
+				console.log("freeing comfy????");
+
+				// Add the workflow into the queue
+				const wasFreed = await ComfyUI.getInstance.free({ unloadModels, freeMemory });
+				console.log("freeing comfy::::", { wasFreed });
+
+				// Send the promptId back to the sender
+				event.sender.send(channel, {
+					action: "comfyui:wasFreed",
+					payload: { wasFreed },
+				});
+
+				break;
+			}
+
 			case "comfyui:registerClient": {
 				// Register the client and receive updates from ComfyUI
 				ComfyUI.getInstance.registerClient(appId, data => {
@@ -48,3 +73,7 @@ ipcMain.on(
 		}
 	}
 );
+
+ipcMain.on(buildKey([ID.COMFYUI_PORT], { suffix: ":get" }), event => {
+	event.sender.send(buildKey([ID.COMFYUI_PORT], { suffix: ":get" }), PORTS.COMFYUI.toString());
+});