- feature: implement image capture

apps/shinkai-tray/src/components/chat/message.tsx

@@ -70,7 +70,7 @@ const Message = ({
       </Avatar>
       <div
         className={cn(
-          'group flex items-start gap-1 break-words rounded-lg bg-transparent px-2.5 py-3',
+          'group flex items-start gap-1 break-words rounded-lg bg-transparent px-2.5 py-3 overflow-x-hidden',
           message.isLocal
             ? 'rounded-tl-none border border-slate-800'
             : 'rounded-tr-none border-none bg-[rgba(217,217,217,0.04)]',

apps/shinkai-visor/public/manifest.json

@@ -37,15 +37,15 @@
       "description": "Open/Close Shinkai popup"
     }
   },
-  "permissions": ["storage", "contextMenus", "scripting"],
+  "permissions": ["storage", "contextMenus", "scripting", "activeTab"],
   "host_permissions": ["<all_urls>"],
   "content_security_policy": {
     "extension_pages": "script-src 'self' 'wasm-unsafe-eval'; object-src 'self';"
   },
   "content_scripts": [
     {
       "matches": ["https://*/*", "http://*/*", "<all_urls>"],
-      "js": ["src/components/action-button/action-button.tsx", "src/components/popup/popup-embeder.ts"]
+      "js": ["src/components/action-button/action-button.tsx", "src/components/popup/popup-embeder.ts", "src/components/image-capture/image-capture.tsx"]
     }
   ],
   "externally_connectable": {

apps/shinkai-visor/src/components/add-agent/models.ts

@@ -14,6 +14,10 @@ export const modelsConfig = {
       {
         name: 'GPT 4 Turbo',
         value: 'gpt-4-1106-preview',
+      },
+      {
+        name: 'GPT 4 Vision',
+        value: 'gpt-4-vision-preview'
       }
     ],
   },

apps/shinkai-visor/src/components/agents/agents.tsx

@@ -35,7 +35,7 @@ export const Agents = () => {
         </div>
       ) : (
         <>
-          <ScrollArea className="flex h-full flex-col justify-between  [&>div>div]:!block">
+          <ScrollArea className="flex h-full flex-col justify-between [&>div>div]:!block">
             <div className="space-y-3">
               {agents?.map((agent) => (
                 <Fragment key={agent.id}>

apps/shinkai-visor/src/components/create-job/create-job.tsx

@@ -31,8 +31,8 @@ import { Header } from '../header/header';
 import { ScrollArea } from '../ui/scroll-area';
 
 const formSchema = z.object({
-  agent: z.string().nonempty(),
-  content: z.string().nonempty(),
+  agent: z.string().min(1),
+  content: z.string().min(1),
   files: z.array(z.any()).max(3),
 });
 
@@ -151,7 +151,7 @@ export const CreateJob = () => {
           className="flex grow flex-col justify-between space-y-2 overflow-hidden"
           onSubmit={form.handleSubmit(submit)}
         >
-          <ScrollArea className="[&>div>div]:!block">
+          <ScrollArea className="pr-4 [&>div>div]:!block">
             <FormField
               control={form.control}
               name="agent"

apps/shinkai-visor/src/components/file-list/file-list.tsx

@@ -1,27 +1,67 @@
 import { PaperClipIcon } from '@shinkai_network/shinkai-ui';
 import { partial } from 'filesize';
 import { AnimatePresence, motion } from 'framer-motion';
-import { ReactNode } from 'react';
+import { Loader2 } from 'lucide-react';
+import { ReactNode, useEffect, useState } from 'react';
 
 import { cn } from '../../helpers/cn-utils';
 import { getFileExt, getFileName } from '../../helpers/file-name-utils';
 
 export type FileListProps = {
-  files: { name: string; size?: number }[];
+  files: ({ name: string; size?: number } | File)[];
   actions: {
     render: ReactNode;
     onClick?: (index: number) => void;
   }[];
   className?: string;
 };
 
+interface FileImagePreview extends React.HTMLAttributes<HTMLDivElement> {
+  file: File;
+}
+
+const FileImagePreview = ({ file, ...props }: FileImagePreview) => {
+  const [imageSrc, setImageSrc] = useState('');
+  useEffect(() => {
+    const reader = new FileReader();
+    reader.addEventListener(
+      'load',
+      function () {
+        setImageSrc(reader.result as string);
+      },
+      false,
+    );
+    if (file) {
+      reader.readAsDataURL(file);
+    }
+  }, [file]);
+  return imageSrc ? (
+    <img alt="preview" src={imageSrc} {...props} />
+  ) : (
+    <Loader2 />
+  );
+};
 export const FileList = ({ files, actions, className }: FileListProps) => {
   const size = partial({ standard: 'jedec' });
   const animations = {
     initial: { scale: 0, opacity: 0 },
     animate: { scale: 1, opacity: 1 },
     exit: { scale: 0, opacity: 0 },
   };
+  const hasPreview = (file: File): boolean => {
+    return file?.type?.includes('image/');
+  };
+  const getFilePreview = (file: File): ReactNode | undefined => {
+    console.log('type', file.type);
+    if (file?.type?.includes('image/')) {
+      return (
+        <FileImagePreview
+          className="h-full rounded-lg object-cover"
+          file={file}
+        />
+      );
+    }
+  };
   return (
     <ul
       className={cn(
@@ -36,38 +76,42 @@ export const FileList = ({ files, actions, className }: FileListProps) => {
             className="flex items-center justify-between p-2 text-sm leading-6"
             key={index}
           >
-            <div className="flex w-0 flex-1 items-center space-x-2">
-              <PaperClipIcon className="h-4 w-4" />
-              <div className="ml-4 flex min-w-0 flex-1 justify-between gap-2">
-                <div className="flex flex-row overflow-hidden">
-                  <span className="text-gray-80 truncate font-medium">
-                    {getFileName(decodeURIComponent(file.name))}
-                  </span>
-                  <span className="text-gray-80 rounded-md bg-gray-200 px-1 text-[10px] font-medium uppercase">
+            <div className="flex w-full flex-col space-y-2 overflow-hidden">
+              {file instanceof File && hasPreview(file) && (
+                <div className="h-10 self-center">{getFilePreview(file)}</div>
+              )}
+              <div className="flex min-w-0 flex-1 flex-row items-center justify-between gap-2">
+                <PaperClipIcon className="h-4 w-4" />
+
+                <span className="text-gray-80 grow truncate font-medium">
+                  {getFileName(decodeURIComponent(file.name))}
+                </span>
+                <div className="flex shrink-0 flex-row space-x-1 overflow-hidden">
+                  <span className="text-gray-80 w-[40px] rounded-md bg-gray-200 px-1 text-center text-[10px] font-medium uppercase">
                     {getFileExt(decodeURIComponent(file.name))}
                   </span>
+                  {file.size && (
+                    <span className="w-[70px] shrink-0 text-center text-gray-200">
+                      {size(file.size)}
+                    </span>
+                  )}
+                  <div className="flex flex-row items-center space-x-1">
+                    {actions?.map((action, actionIndex) => {
+                      return (
+                        <div
+                          key={actionIndex}
+                          onClick={() => {
+                            if (typeof action.onClick === 'function') {
+                              action.onClick(index);
+                            }
+                          }}
+                        >
+                          {action.render}
+                        </div>
+                      );
+                    })}
+                  </div>
                 </div>
-                {file.size && (
-                  <span className="flex-shrink-0 text-gray-400">
-                    {size(file.size)}
-                  </span>
-                )}
-              </div>
-              <div className="flex flex-row space-x-1">
-                {actions?.map((action, actionIndex) => {
-                  return (
-                    <div
-                      key={actionIndex}
-                      onClick={() => {
-                        if (typeof action.onClick === 'function') {
-                          action.onClick(index);
-                        }
-                      }}
-                    >
-                      {action.render}
-                    </div>
-                  );
-                })}
               </div>
             </div>
           </motion.li>

apps/shinkai-visor/src/components/image-capture/image-capture.tsx

@@ -0,0 +1,94 @@
+import * as React from 'react';
+import { useRef, useState } from 'react';
+import { createRoot } from 'react-dom/client';
+import ReactCrop, { Crop, PixelCrop } from 'react-image-crop';
+import reactCropStyle from 'react-image-crop/dist/ReactCrop.css?inline';
+import { IntlProvider } from 'react-intl';
+
+import { blobToBase64 } from '../../helpers/blob-utils';
+import { canvasPreview, canvasToBlob } from '../../helpers/canvas-utils';
+import { useGlobalImageCaptureChromeMessage } from '../../hooks/use-global-image-capture-message';
+import { langMessages, locale } from '../../lang/intl';
+import themeStyle from '../../theme/styles.css?inline';
+
+const baseContainer = document.createElement('shinkai-image-capture-root');
+const shadow = baseContainer.attachShadow({ mode: 'open' });
+const container = document.createElement('div');
+container.id = 'root';
+shadow.appendChild(container);
+const htmlRoot = document.getElementsByTagName('html')[0];
+htmlRoot.prepend(baseContainer);
+
+export const ImageCapture = () => {
+  const [baseImage, setBaseImage] = useState<string | undefined>(undefined);
+  const [crop, setCrop] = useState<Crop>();
+  const imageRef= useRef<HTMLImageElement | null>(null);
+  const finishCaptureRef = useRef<(image: string) => void>();
+  useGlobalImageCaptureChromeMessage({
+    capture: ({ image: baseImage, finishCapture }) => {
+      setBaseImage(baseImage);
+      finishCaptureRef.current = finishCapture;
+    },
+  });
+  const getCroppedImage = async (
+    image: HTMLImageElement,
+    crop: PixelCrop,
+    scale = 1,
+    rotate = 0,
+  ): Promise<Blob | null> => {
+    const canvas = document.createElement('canvas');
+    await canvasPreview(image, canvas, crop, scale, rotate);
+    const blob = await canvasToBlob(canvas, 'image/jpeg', 0.5);
+    if (!blob) {
+      console.error('Failed to create blob');
+      return null;
+    }
+    return blob;
+  };
+  return baseImage && (
+    <ReactCrop
+      className="h-full w-full"
+      crop={crop}
+      onChange={(_, percentCrop) => {
+        console.log('crop change', _, percentCrop);
+        setCrop(percentCrop);
+      }}
+      onComplete={async (c) => {
+        console.log('crop completed', c);
+        if (typeof finishCaptureRef.current === 'function' && imageRef.current) {
+          if (c.width === 0 || c.height === 0) {
+            return;
+          }
+          const croppedImage = await getCroppedImage(imageRef.current, c, 1, 0);
+          if (!croppedImage) {
+            return;
+          }
+          const croppedImageAsBase64 = await blobToBase64(croppedImage);
+          finishCaptureRef.current(croppedImageAsBase64);
+          setBaseImage('');
+          setCrop(undefined);
+        }
+      }}
+      style={{pointerEvents: 'all'}}
+    >
+      <img
+        alt="capture-placeholder"
+        className="h-full w-full invisible"
+        ref={imageRef}
+        src={`${baseImage}`}
+      ></img>
+    </ReactCrop>
+  );
+};
+const root = createRoot(container);
+root.render(
+  <React.StrictMode>
+    <style>{themeStyle}</style>
+    <style>{reactCropStyle}</style>
+    <IntlProvider locale={locale} messages={langMessages}>
+      <div className="fixed z-[2000000000] h-full w-full overflow-hidden pointer-events-none">
+        <ImageCapture></ImageCapture>
+      </div>
+    </IntlProvider>
+  </React.StrictMode>,
+);

apps/shinkai-visor/src/components/inbox/inbox.tsx

@@ -180,7 +180,7 @@ export const Inbox = () => {
 
   return (
     <div className="flex h-full flex-col justify-between space-y-3">
-      <ScrollArea className="h-full pr-4" ref={chatContainerRef}>
+      <ScrollArea className="h-full pr-4 [&>div>div]:!block" ref={chatContainerRef}>
         {isChatConversationSuccess && (
           <div className="py-2 text-center text-xs">
             {isFetchingPreviousPage && (
@@ -192,28 +192,31 @@ export const Inbox = () => {
           </div>
         )}
         <div className="">
-          {isChatConversationLoading &&
-            [...Array(5).keys()].map((index) => (
-              <div
-                className={cn(
-                  'flex w-[95%] items-start gap-3',
-                  index % 2 === 0
-                    ? 'ml-0 mr-auto flex-row'
-                    : 'ml-auto mr-0 flex-row-reverse',
-                )}
-                key={`${index}`}
-              >
-                <Skeleton className="h-12 w-12 rounded-full" key={index} />
-                <Skeleton
+          {isChatConversationLoading && (
+            <div className="flex flex-col space-y-2">
+              {[...Array(5).keys()].map((index) => (
+                <div
                   className={cn(
-                    'w-full rounded-lg px-2.5 py-3',
+                    'flex w-[95%] items-start gap-3',
                     index % 2 === 0
-                      ? 'rounded-tl-none border border-slate-800'
-                      : 'rounded-tr-none border-none',
+                      ? 'ml-0 mr-auto flex-row'
+                      : 'ml-auto mr-0 flex-row-reverse',
                   )}
-                />
-              </div>
-            ))}
+                  key={`${index}`}
+                >
+                  <Skeleton className="shrink-0 h-12 w-12 rounded-full" key={index} />
+                  <Skeleton
+                    className={cn(
+                      'w-full h-16 rounded-lg px-2.5 py-3',
+                      index % 2 === 0
+                        ? 'rounded-tl-none border border-slate-800'
+                        : 'rounded-tr-none border-none',
+                    )}
+                  />
+                </div>
+              ))}
+            </div>
+          )}
           {isChatConversationSuccess &&
             data?.pages?.map((group, index) => (
               <Fragment key={index}>
@@ -223,7 +226,7 @@ export const Inbox = () => {
                       <div key={date}>
                         <div
                           className={cn(
-                            'relative z-10 m-auto flex h-[30px] w-[70px] items-center justify-center rounded-xl bg-gray-400',
+                            'relative z-10 m-auto flex h-[30px] w-[150px] items-center justify-center rounded-xl bg-gray-400',
                             true && 'sticky top-5',
                           )}
                         >

apps/shinkai-visor/src/components/inboxes/inboxes.tsx

@@ -109,7 +109,7 @@ export const Inboxes = () => {
       ) : (
         <>
           <div className="flex grow flex-col overflow-hidden">
-            <ScrollArea className="[&>div>div]:!block">
+            <ScrollArea className="pr-4 [&>div>div]:!block">
               <div className="space-y-4">
                 {inboxes?.map((inbox) => (
                   <Fragment key={inbox.inbox_id}>