{agents?.map((agent) => (
diff --git a/apps/shinkai-visor/src/components/create-job/create-job.tsx b/apps/shinkai-visor/src/components/create-job/create-job.tsx
index 1b37337e7..f14560744 100644
--- a/apps/shinkai-visor/src/components/create-job/create-job.tsx
+++ b/apps/shinkai-visor/src/components/create-job/create-job.tsx
@@ -31,8 +31,8 @@ import { Header } from '../header/header';
import { ScrollArea } from '../ui/scroll-area';
const formSchema = z.object({
- agent: z.string().nonempty(),
- content: z.string().nonempty(),
+ agent: z.string().min(1),
+ content: z.string().min(1),
files: z.array(z.any()).max(3),
});
@@ -151,7 +151,7 @@ export const CreateJob = () => {
className="flex grow flex-col justify-between space-y-2 overflow-hidden"
onSubmit={form.handleSubmit(submit)}
>
-
+
void;
@@ -15,6 +16,31 @@ export type FileListProps = {
className?: string;
};
+interface FileImagePreview extends React.HTMLAttributes {
+ file: File;
+}
+
+const FileImagePreview = ({ file, ...props }: FileImagePreview) => {
+ const [imageSrc, setImageSrc] = useState('');
+ useEffect(() => {
+ const reader = new FileReader();
+ reader.addEventListener(
+ 'load',
+ function () {
+ setImageSrc(reader.result as string);
+ },
+ false,
+ );
+ if (file) {
+ reader.readAsDataURL(file);
+ }
+ }, [file]);
+ return imageSrc ? (
+ 
+ ) : (
+
- {
className="flex items-center justify-between p-2 text-sm leading-6"
key={index}
>
-
-
-
-
- {getFileName(decodeURIComponent(file.name))}
-
-
+ (undefined);
+ const [crop, setCrop] = useState();
+ const imageRef= useRef(null);
+ const finishCaptureRef = useRef<(image: string) => void>();
+ useGlobalImageCaptureChromeMessage({
+ capture: ({ image: baseImage, finishCapture }) => {
+ setBaseImage(baseImage);
+ finishCaptureRef.current = finishCapture;
+ },
+ });
+ const getCroppedImage = async (
+ image: HTMLImageElement,
+ crop: PixelCrop,
+ scale = 1,
+ rotate = 0,
+ ): Promise => {
+ const canvas = document.createElement('canvas');
+ await canvasPreview(image, canvas, crop, scale, rotate);
+ const blob = await canvasToBlob(canvas, 'image/jpeg', 0.5);
+ if (!blob) {
+ console.error('Failed to create blob');
+ return null;
+ }
+ return blob;
+ };
+ return baseImage && (
+ {
+ console.log('crop change', _, percentCrop);
+ setCrop(percentCrop);
+ }}
+ onComplete={async (c) => {
+ console.log('crop completed', c);
+ if (typeof finishCaptureRef.current === 'function' && imageRef.current) {
+ if (c.width === 0 || c.height === 0) {
+ return;
+ }
+ const croppedImage = await getCroppedImage(imageRef.current, c, 1, 0);
+ if (!croppedImage) {
+ return;
+ }
+ const croppedImageAsBase64 = await blobToBase64(croppedImage);
+ finishCaptureRef.current(croppedImageAsBase64);
+ setBaseImage('');
+ setCrop(undefined);
+ }
+ }}
+ style={{pointerEvents: 'all'}}
+ >
+ 
+
+ );
+};
+const root = createRoot(container);
+root.render(
+
+
+
+
+
+ ,
+);
diff --git a/apps/shinkai-visor/src/components/inbox/inbox.tsx b/apps/shinkai-visor/src/components/inbox/inbox.tsx
index 66aeebe22..a784c6356 100644
--- a/apps/shinkai-visor/src/components/inbox/inbox.tsx
+++ b/apps/shinkai-visor/src/components/inbox/inbox.tsx
@@ -180,7 +180,7 @@ export const Inbox = () => {
return (
@@ -223,7 +226,7 @@ export const Inbox = () => {
+ {file instanceof File && hasPreview(file) && (
+
diff --git a/apps/shinkai-visor/src/components/image-capture/image-capture.tsx b/apps/shinkai-visor/src/components/image-capture/image-capture.tsx
new file mode 100644
index 000000000..44d0ce635
--- /dev/null
+++ b/apps/shinkai-visor/src/components/image-capture/image-capture.tsx
@@ -0,0 +1,94 @@
+import * as React from 'react';
+import { useRef, useState } from 'react';
+import { createRoot } from 'react-dom/client';
+import ReactCrop, { Crop, PixelCrop } from 'react-image-crop';
+import reactCropStyle from 'react-image-crop/dist/ReactCrop.css?inline';
+import { IntlProvider } from 'react-intl';
+
+import { blobToBase64 } from '../../helpers/blob-utils';
+import { canvasPreview, canvasToBlob } from '../../helpers/canvas-utils';
+import { useGlobalImageCaptureChromeMessage } from '../../hooks/use-global-image-capture-message';
+import { langMessages, locale } from '../../lang/intl';
+import themeStyle from '../../theme/styles.css?inline';
+
+const baseContainer = document.createElement('shinkai-image-capture-root');
+const shadow = baseContainer.attachShadow({ mode: 'open' });
+const container = document.createElement('div');
+container.id = 'root';
+shadow.appendChild(container);
+const htmlRoot = document.getElementsByTagName('html')[0];
+htmlRoot.prepend(baseContainer);
+
+export const ImageCapture = () => {
+ const [baseImage, setBaseImage] = useState{getFilePreview(file)}
+ )}
+
+
-
+
{getFileExt(decodeURIComponent(file.name))}
+ {file.size && (
+
+ {size(file.size)}
+
+ )}
+
- {file.size && (
-
- {size(file.size)}
-
- )}
-
+ {actions?.map((action, actionIndex) => {
+ return (
+
{
+ if (typeof action.onClick === 'function') {
+ action.onClick(index);
+ }
+ }}
+ >
+ {action.render}
+
+ );
+ })}
+
- {actions?.map((action, actionIndex) => {
- return (
-
{
- if (typeof action.onClick === 'function') {
- action.onClick(index);
- }
- }}
- >
- {action.render}
-
- );
- })}
+
+
-
+
{isChatConversationSuccess && (
+ )}
{isChatConversationSuccess &&
data?.pages?.map((group, index) => (
{isFetchingPreviousPage && (
@@ -192,28 +192,31 @@ export const Inbox = () => {
)}
- {isChatConversationLoading &&
- [...Array(5).keys()].map((index) => (
-
+ ))}
+
-
+ {[...Array(5).keys()].map((index) => (
+
-
- ))}
+ key={`${index}`}
+ >
+
diff --git a/apps/shinkai-visor/src/components/inboxes/inboxes.tsx b/apps/shinkai-visor/src/components/inboxes/inboxes.tsx
index 5f5b3c889..b539f7d34 100644
--- a/apps/shinkai-visor/src/components/inboxes/inboxes.tsx
+++ b/apps/shinkai-visor/src/components/inboxes/inboxes.tsx
@@ -109,7 +109,7 @@ export const Inboxes = () => {
) : (
<>
-
+
{inboxes?.map((inbox) => (
diff --git a/apps/shinkai-visor/src/components/message/message.tsx b/apps/shinkai-visor/src/components/message/message.tsx
index b26d37e1c..cb3c09fde 100644
--- a/apps/shinkai-visor/src/components/message/message.tsx
+++ b/apps/shinkai-visor/src/components/message/message.tsx
@@ -40,7 +40,7 @@ export const Message = ({ message }: MessageProps) => {
{
>
{message.isLocal ? null : (
copyToClipboard(message.content)}
string={message.content}
/>
@@ -61,7 +61,7 @@ export const Message = ({ message }: MessageProps) => {
{!!message.fileInbox?.files?.length && (
=> {
+ const reader = new FileReader();
+ reader.readAsDataURL(blob);
+ return new Promise(resolve => {
+ reader.onloadend = () => {
+ resolve(reader.result as string);
+ };
+ });
+};
+
+export const dataUrlToFile = (dataUrl: string, filename: string): File | undefined => {
+ const arr = dataUrl.split(',');
+ if (arr.length < 2) { return undefined; }
+ const mimeArr = arr[0].match(/:(.*?);/);
+ if (!mimeArr || mimeArr.length < 2) { return undefined; }
+ const mime = mimeArr[1];
+ const buff = Buffer.from(arr[1], 'base64');
+ return new File([buff], filename, {type:mime});
+};
diff --git a/apps/shinkai-visor/src/helpers/canvas-utils.ts b/apps/shinkai-visor/src/helpers/canvas-utils.ts
new file mode 100644
index 000000000..87eb8aea9
--- /dev/null
+++ b/apps/shinkai-visor/src/helpers/canvas-utils.ts
@@ -0,0 +1,70 @@
+import { PixelCrop } from "react-image-crop"
+const TO_RADIANS = Math.PI / 180
+
+export async function canvasPreview(
+ image: HTMLImageElement,
+ canvas: HTMLCanvasElement,
+ crop: PixelCrop,
+ scale = 1,
+ rotate = 0,
+) {
+ const ctx = canvas.getContext('2d')
+
+ if (!ctx) {
+ throw new Error('No 2d context')
+ }
+
+ const scaleX = image.naturalWidth / image.width
+ const scaleY = image.naturalHeight / image.height
+ // devicePixelRatio slightly increases sharpness on retina devices
+ // at the expense of slightly slower render times and needing to
+ // size the image back down if you want to download/upload and be
+ // true to the images natural size.
+ const pixelRatio = window.devicePixelRatio
+ // const pixelRatio = 1
+
+ canvas.width = Math.floor(crop.width * scaleX * pixelRatio)
+ canvas.height = Math.floor(crop.height * scaleY * pixelRatio)
+
+ ctx.scale(pixelRatio, pixelRatio)
+ ctx.imageSmoothingQuality = 'high'
+
+ const cropX = crop.x * scaleX
+ const cropY = crop.y * scaleY
+
+ const rotateRads = rotate * TO_RADIANS
+ const centerX = image.naturalWidth / 2
+ const centerY = image.naturalHeight / 2
+
+ ctx.save()
+
+ // 5) Move the crop origin to the canvas origin (0,0)
+ ctx.translate(-cropX, -cropY)
+ // 4) Move the origin to the center of the original position
+ ctx.translate(centerX, centerY)
+ // 3) Rotate around the origin
+ ctx.rotate(rotateRads)
+ // 2) Scale the image
+ ctx.scale(scale, scale)
+ // 1) Move the center of the image to the origin (0,0)
+ ctx.translate(-centerX, -centerY)
+ ctx.drawImage(
+ image,
+ 0,
+ 0,
+ image.naturalWidth,
+ image.naturalHeight,
+ 0,
+ 0,
+ image.naturalWidth,
+ image.naturalHeight,
+ )
+
+ ctx.restore()
+}
+
+export const canvasToBlob = (canvas: HTMLCanvasElement, type: 'image/jpeg' = 'image/jpeg', quality: number = 0.92): Promise => {
+ return new Promise((resolve) => {
+ canvas.toBlob(resolve, 'image/jpeg', 0.5);
+ })
+};
diff --git a/apps/shinkai-visor/src/hooks/use-chrome-message.ts b/apps/shinkai-visor/src/hooks/use-chrome-message.ts
index 3b8900b0a..410ed77ea 100644
--- a/apps/shinkai-visor/src/hooks/use-chrome-message.ts
+++ b/apps/shinkai-visor/src/hooks/use-chrome-message.ts
@@ -3,19 +3,20 @@ import { useEffect } from "react";
import { ServiceWorkerInternalMessage } from "../service-worker/communication/internal/types";
-export type UseChromeMessageCallbackParameters = [message: ServiceWorkerInternalMessage, sender: chrome.runtime.MessageSender];
-export type UseChromeMessageCallback = (...params: UseChromeMessageCallbackParameters) => void;
+export type UseChromeMessageCallbackParameters = [message: ServiceWorkerInternalMessage, sender: chrome.runtime.MessageSender, sendResponse: (response?: unknown) => void];
+export type UseChromeMessageCallback = (...params: UseChromeMessageCallbackParameters) => Promise | unknown;
export const useChromeMessage = (callback: UseChromeMessageCallback) => {
useEffect(() => {
- function onMessage(message: ServiceWorkerInternalMessage, sender: chrome.runtime.MessageSender): void {
+ function onMessage(message: ServiceWorkerInternalMessage, sender: chrome.runtime.MessageSender, sendResponse: (response?: unknown) => void): void | boolean {
if (sender.tab) {
return;
}
console.info('on chrome message', window.location.href, message, sender);
if (typeof callback === 'function') {
- callback(message, sender);
+ callback(message, sender, sendResponse);
}
+ return true;
};
chrome.runtime.onMessage.removeListener(onMessage);
chrome.runtime.onMessage.addListener(onMessage);
diff --git a/apps/shinkai-visor/src/hooks/use-global-action-button-chrome-message.ts b/apps/shinkai-visor/src/hooks/use-global-action-button-chrome-message.ts
index e485dfbd9..1b51c0c41 100644
--- a/apps/shinkai-visor/src/hooks/use-global-action-button-chrome-message.ts
+++ b/apps/shinkai-visor/src/hooks/use-global-action-button-chrome-message.ts
@@ -7,7 +7,7 @@ import { useChromeMessage } from "./use-chrome-message";
export const useGlobalActionButtonChromeMessage = () => {
const [popupVisibility, setPopupVisibility] = useState(false);
- useChromeMessage((message, sender) => {
+ useChromeMessage((message) => {
switch (message.type) {
case ServiceWorkerInternalMessageType.ContentScriptBridge:
switch (message.data.type) {
diff --git a/apps/shinkai-visor/src/hooks/use-global-image-capture-message.ts b/apps/shinkai-visor/src/hooks/use-global-image-capture-message.ts
new file mode 100644
index 000000000..ba81d57a6
--- /dev/null
+++ b/apps/shinkai-visor/src/hooks/use-global-image-capture-message.ts
@@ -0,0 +1,19 @@
+import { ServiceWorkerInternalMessageType } from "../service-worker/communication/internal/types";
+import { useChromeMessage } from "./use-chrome-message";
+
+export const useGlobalImageCaptureChromeMessage = ({
+ capture
+}: {
+ capture: ({image, finishCapture }: { image: string, finishCapture: (image: string) => void }) => void
+}) => {
+ useChromeMessage(async (message, sender, sendResponse) => {
+ switch (message.type) {
+ case ServiceWorkerInternalMessageType.CaptureImage: {
+ capture({ image: message.data.image, finishCapture: (image) => sendResponse(image) });
+ break;
+ }
+ default:
+ break;
+ }
+ });
+};
diff --git a/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts b/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts
index 5324cc0b4..abf4a61b5 100644
--- a/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts
+++ b/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts
@@ -1,6 +1,7 @@
import { useState } from "react";
import { useHistory } from "react-router-dom";
+import { dataUrlToFile } from "../helpers/blob-utils";
import { sendContentScriptMessage } from "../service-worker/communication/internal";
import { ContentScriptBridgeMessageType, ServiceWorkerInternalMessageType } from "../service-worker/communication/internal/types";
import { useAuth } from "../store/auth/auth";
@@ -22,6 +23,12 @@ export const useGlobalPopupChromeMessage = () => {
history.push({ pathname: '/inboxes/create-job', state: { files: [message.data.pdf] } });
sendContentScriptMessage({ type: ContentScriptBridgeMessageType.TogglePopupVisibility, data: true });
break;
+ case ServiceWorkerInternalMessageType.SendCaptureToAgent: {
+ const imageFile = dataUrlToFile(message.data.image, message.data.filename);
+ history.push({ pathname: '/inboxes/create-job', state: { files: [imageFile] } });
+ sendContentScriptMessage({ type: ContentScriptBridgeMessageType.TogglePopupVisibility, data: true });
+ break;
+ }
case ServiceWorkerInternalMessageType.ContentScriptBridge:
switch (message.data.type) {
case ContentScriptBridgeMessageType.TogglePopupVisibility:
diff --git a/apps/shinkai-visor/src/service-worker/communication/internal/types.ts b/apps/shinkai-visor/src/service-worker/communication/internal/types.ts
index b8b503b63..b77df5b7c 100644
--- a/apps/shinkai-visor/src/service-worker/communication/internal/types.ts
+++ b/apps/shinkai-visor/src/service-worker/communication/internal/types.ts
@@ -4,6 +4,8 @@ export enum ServiceWorkerInternalMessageType {
SendPageToAgent = 'send-page-to-agent',
RehydrateStore = 'rehydrate-store',
CopyToClipboard = 'copy-to-clipboard',
+ CaptureImage = 'capture-image',
+ SendCaptureToAgent = 'send-capture-to-agent',
}
export enum ContentScriptBridgeMessageType {
@@ -25,8 +27,11 @@ export type ServiceWorkerInternalMessage =
type: ServiceWorkerInternalMessageType.SendPageToAgent;
data: {
pdf?: File;
+ filename: string;
}
}
| { type: ServiceWorkerInternalMessageType.RehydrateStore, data?: never }
- | { type: ServiceWorkerInternalMessageType.CopyToClipboard, data: { content: string } };
+ | { type: ServiceWorkerInternalMessageType.CopyToClipboard, data: { content: string } }
+ | { type: ServiceWorkerInternalMessageType.CaptureImage, data: { image: string } }
+ | { type: ServiceWorkerInternalMessageType.SendCaptureToAgent, data: { image: string, filename: string } };
;
diff --git a/apps/shinkai-visor/src/service-worker/context-menu.ts b/apps/shinkai-visor/src/service-worker/context-menu.ts
index 27163facc..e83982d65 100644
--- a/apps/shinkai-visor/src/service-worker/context-menu.ts
+++ b/apps/shinkai-visor/src/service-worker/context-menu.ts
@@ -3,6 +3,7 @@ import { ServiceWorkerInternalMessage, ServiceWorkerInternalMessageType } from "
enum ContextMenu {
SendPageToAgent = 'send-page-to-agent',
SendToAgent = 'send-to-agent',
+ SendCaptureToAgent = 'send-capture-to-agent',
}
const sendPageToAgent = async (info: chrome.contextMenus.OnClickData, tab: chrome.tabs.Tab | undefined) => {
@@ -12,9 +13,11 @@ const sendPageToAgent = async (info: chrome.contextMenus.OnClickData, tab: chrom
}
const message: ServiceWorkerInternalMessage = {
type: ServiceWorkerInternalMessageType.SendPageToAgent,
- data: {},
+ data: {
+ filename: `${encodeURIComponent(tab.url || Date.now())}.pdf`
+ },
};
- chrome.tabs.sendMessage(tab.id, message);
+ chrome.tabs.sendMessage(tab.id, message);
}
const sendToAgent = (info: chrome.contextMenus.OnClickData, tab: chrome.tabs.Tab | undefined) => {
@@ -31,9 +34,34 @@ const sendToAgent = (info: chrome.contextMenus.OnClickData, tab: chrome.tabs.Tab
chrome.tabs.sendMessage(tab.id, message);
}
+const sendCaptureToAgent = async (info: chrome.contextMenus.OnClickData, tab: chrome.tabs.Tab | undefined) => {
+ if (!tab?.id) {
+ return;
+ }
+ const image = await new Promise((resolve) => {
+ chrome.tabs.query({ active: true, currentWindow: true }, ([tab]) => {
+ chrome.tabs.captureVisibleTab(tab.windowId, { format: 'jpeg', quality: 92 }, (image) => {
+ resolve(image);
+ });
+ })
+ });
+ let message: ServiceWorkerInternalMessage = {
+ type: ServiceWorkerInternalMessageType.CaptureImage,
+ data: { image },
+ };
+ const croppedImage = await chrome.tabs.sendMessage(tab.id, message);
+ console.log('cropped image', croppedImage);
+ message = {
+ type: ServiceWorkerInternalMessageType.SendCaptureToAgent,
+ data: { image: croppedImage, filename: `${encodeURIComponent(tab.url || 'capture')}.jpeg` },
+ };
+ chrome.tabs.sendMessage(tab.id, message);
+}
+
const menuActions = new Map void>([
[ContextMenu.SendPageToAgent, sendPageToAgent],
[ContextMenu.SendToAgent, sendToAgent],
+ [ContextMenu.SendCaptureToAgent, sendCaptureToAgent],
]);
const registerMenu = () => {
@@ -49,6 +77,13 @@ const registerMenu = () => {
title: 'Send Selection to Agent',
contexts: ['selection']
});
+ chrome.contextMenus.create(
+ {
+ id: ContextMenu.SendCaptureToAgent,
+ title: 'Send Capture to Agent',
+ contexts: ['all']
+ }
+ );
}
chrome.runtime.onInstalled.addListener(() => {
diff --git a/package-lock.json b/package-lock.json
index 8df18f8e8..141946bfc 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -72,6 +72,7 @@
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.47.0",
+ "react-image-crop": "^10.1.8",
"react-intl": "^6.4.5",
"react-router": "^5.3.4",
"react-router-dom": "^5.3.4",
@@ -28001,6 +28002,14 @@
"react": "^16.8.0 || ^17 || ^18"
}
},
+ "node_modules/react-image-crop": {
+ "version": "10.1.8",
+ "resolved": "https://registry.npmjs.org/react-image-crop/-/react-image-crop-10.1.8.tgz",
+ "integrity": "sha512-4rb8XtXNx7ZaOZarKKnckgz4xLMvds/YrU6mpJfGhGAsy2Mg4mIw1x+DCCGngVGq2soTBVVOxx2s/C6mTX9+pA==",
+ "peerDependencies": {
+ "react": ">=16.13.1"
+ }
+ },
"node_modules/react-intl": {
"version": "6.5.5",
"resolved": "https://registry.npmjs.org/react-intl/-/react-intl-6.5.5.tgz",
diff --git a/package.json b/package.json
index a2ffb616f..b0eb0eb94 100644
--- a/package.json
+++ b/package.json
@@ -153,6 +153,7 @@
"react": "18.2.0",
"react-dom": "18.2.0",
"react-hook-form": "^7.47.0",
+ "react-image-crop": "^10.1.8",
"react-intl": "^6.4.5",
"react-router": "^5.3.4",
"react-router-dom": "^5.3.4",