feat: add summarize action and shortcut to send message (#132)

* fix: sidebar icon * fix: support Mod + Enter shortcut to send message * display back context actions * update icons * feat: submit form if initialText is set * fix: wording * fix eslint
dcSpark · Feb 6, 2024 · 207595c · 207595c
1 parent fcc5fe3
commit 207595c
Show file tree

Hide file tree

Showing 11 changed files with 121 additions and 25 deletions.
diff --git a/apps/shinkai-visor/public/manifest.json b/apps/shinkai-visor/public/manifest.json
@@ -56,7 +56,8 @@
     {
       "matches": ["https://*/*", "http://*/*", "<all_urls>"],
       "js": [
-        "src/components/image-capture/image-capture.tsx"
+        "src/components/image-capture/image-capture.tsx",
+        "src/components/action-button/action-button.tsx"
       ]
     }
   ],

diff --git a/apps/shinkai-visor/src/components/action-button/action-button.tsx b/apps/shinkai-visor/src/components/action-button/action-button.tsx
@@ -19,7 +19,7 @@ import {
 } from '@shinkai_network/shinkai-ui';
 import { cn } from '@shinkai_network/shinkai-ui/utils';
 import { motion } from 'framer-motion';
-import { FileInputIcon, ScissorsIcon } from 'lucide-react';
+import { Focus, PanelTopIcon } from 'lucide-react';
 import * as React from 'react';
 import { createRoot } from 'react-dom/client';
 import { IntlProvider } from 'react-intl';
@@ -163,12 +163,12 @@ const ActionButton = () => {
                 {
                   label: 'Send Capture',
                   onClick: sendCapture,
-                  icon: <ScissorsIcon className="h-full w-full" />,
+                  icon: <Focus className="h-full w-full" />,
                 },
                 {
                   label: 'Send Page',
                   onClick: sendPage,
-                  icon: <FileInputIcon className="h-full w-full" />,
+                  icon: <PanelTopIcon className="h-full w-full" />,
                 },
               ].map((item) => (
                 <TooltipProvider key={item.label}>

diff --git a/apps/shinkai-visor/src/components/create-job/create-job.tsx b/apps/shinkai-visor/src/components/create-job/create-job.tsx
@@ -50,7 +50,7 @@ export const CreateJob = () => {
     resolver: zodResolver(formSchema),
     defaultValues: {
       agent: '',
-      content: '',
+      content: query.get('initialText') ?? '',
       files: [],
     },
   });
@@ -102,12 +102,7 @@ export const CreateJob = () => {
     if (!location?.state?.agentName) {
       return;
     }
-    const agent = agents.find(
-      (agent) =>
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        (agent.full_identity_name as any)?.subidentity_name ===
-        location.state.agentName,
-    );
+    const agent = agents.find((agent) => agent.id === location.state.agentName);
     if (agent) {
       form.setValue('agent', agent.id);
     }
@@ -126,6 +121,12 @@ export const CreateJob = () => {
     defaultAgentId = defaultAgentId || (agents?.length ? agents[0].id : '');
     form.setValue('agent', defaultAgentId);
   }, [form, location, agents, currentDefaultAgentId]);
+
+  useEffect(() => {
+    if (query.get('initialText')) {
+      form.handleSubmit(submit)();
+    }
+  }, []);
   const submit = (values: FormSchemaType) => {
     if (!auth) return;
     let content = values.content;

diff --git a/apps/shinkai-visor/src/components/file-input/file-input.tsx b/apps/shinkai-visor/src/components/file-input/file-input.tsx
@@ -1,6 +1,12 @@
 import { cn } from '@shinkai_network/shinkai-ui/utils';
 import { ExternalLink, Trash, Upload } from 'lucide-react';
-import { ChangeEvent, useCallback, useEffect, useRef, useState } from 'react';
+import React, {
+  ChangeEvent,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from 'react';
 import { FormattedMessage } from 'react-intl';
 
 import { getFileExt } from '../../helpers/file-name-utils';
@@ -62,7 +68,7 @@ export const FileInput = ({
     event.preventDefault();
     let draggedFiles: File[] = [];
     if (event.dataTransfer.items) {
-      Array.from(event.dataTransfer.items || []).forEach((item, i) => {
+      Array.from(event.dataTransfer.items || []).forEach((item) => {
         if (item.kind === 'file') {
           const file = item.getAsFile();
           if (!file) {
@@ -91,7 +97,7 @@ export const FileInput = ({
     forceUpdate();
   }, [value, forceUpdate]);
   return (
-    <div className="flex flex-col space-y-1">
+    <div className="flex flex-col space-y-3">
       <div className="flex items-center justify-center">
         <label
           className={cn(

diff --git a/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts b/apps/shinkai-visor/src/hooks/use-global-popup-chrome-message.ts
@@ -31,6 +31,21 @@ export const useGlobalPopupChromeMessage = () => {
         });
         break;
       }
+      case ServiceWorkerInternalMessageType.SummarizePage: {
+        const params = new URLSearchParams({
+          initialText: 'Summarize this',
+        });
+        const file = dataUrlToFile(
+          message.data.fileDataUrl,
+          message.data.filename,
+        );
+        history.push({
+          pathname: '/inboxes/create-job',
+          state: { files: [file] },
+          search: params.toString(),
+        });
+        break;
+      }
       case ServiceWorkerInternalMessageType.SendCaptureToAgent: {
         const imageFile = dataUrlToFile(
           message.data.imageDataUrl,

diff --git a/apps/shinkai-visor/src/service-worker/action.ts b/apps/shinkai-visor/src/service-worker/action.ts
@@ -36,6 +36,34 @@ export const sendPageToAgent = async (
   };
   sendMessage(message);
 };
+export const summarizePage = async (
+  info: chrome.contextMenus.OnClickData | undefined,
+  tab: chrome.tabs.Tab | undefined,
+) => {
+  // At this point, agents can just process text
+  if (!tab?.id) {
+    return;
+  }
+  const [htmlContent] = await chrome.scripting.executeScript({
+    target: { tabId: tab?.id },
+    func: () => {
+      return document.documentElement.outerHTML;
+    },
+    args: [],
+  });
+  const fileType = 'text/html';
+  const message: ServiceWorkerInternalMessage = {
+    type: ServiceWorkerInternalMessageType.SummarizePage,
+    data: {
+      filename: `${encodeURIComponent(tab.url || Date.now())}.html`,
+      fileType: fileType,
+      fileDataUrl: `data:${fileType};base64,${Buffer.from(
+        htmlContent.result,
+      ).toString('base64')}`,
+    },
+  };
+  sendMessage(message);
+};
 export const sendCaptureToAgent = async (
   info: chrome.contextMenus.OnClickData | undefined,
   tab: chrome.tabs.Tab | undefined,

diff --git a/apps/shinkai-visor/src/service-worker/communication/internal/types.ts b/apps/shinkai-visor/src/service-worker/communication/internal/types.ts
@@ -2,6 +2,7 @@ export enum ServiceWorkerInternalMessageType {
   ContentScriptBridge = 'content-script-bridge',
   SendToAgent = 'send-to-agent',
   SendPageToAgent = 'send-page-to-agent',
+  SummarizePage = 'summarize-page',
   RehydrateStore = 'rehydrate-store',
   CopyToClipboard = 'copy-to-clipboard',
   CaptureImage = 'capture-image',
@@ -53,6 +54,14 @@ export type ServiceWorkerInternalMessage =
         fileType: string;
       };
     }
+  | {
+      type: ServiceWorkerInternalMessageType.SummarizePage;
+      data: {
+        filename: string;
+        fileDataUrl: string;
+        fileType: string;
+      };
+    }
   | { type: ServiceWorkerInternalMessageType.RehydrateStore; data?: never }
   | {
       type: ServiceWorkerInternalMessageType.CopyToClipboard;

diff --git a/apps/shinkai-visor/src/service-worker/context-menu.ts b/apps/shinkai-visor/src/service-worker/context-menu.ts
@@ -1,29 +1,61 @@
 import { delay } from '../helpers/misc';
-import { OPEN_SIDEPANEL_DELAY_MS, sendToAgent } from './action';
+import {
+  OPEN_SIDEPANEL_DELAY_MS,
+  sendCaptureToAgent,
+  sendPageToAgent,
+  sendToAgent,
+  summarizePage,
+} from './action';
 
 enum ContextMenu {
+  SendPageToAgent = 'send-page-to-agent',
   SendToAgent = 'send-to-agent',
+  SendCaptureToAgent = 'send-capture-to-agent',
+  SummarizePage = 'summarize-page',
 }
 
 const menuActions = new Map<
   string | number,
   (
-    info: chrome.contextMenus.OnClickData | undefined,
+    info: chrome.contextMenus.OnClickData,
     tab: chrome.tabs.Tab | undefined,
   ) => void
->([[ContextMenu.SendToAgent, sendToAgent]]);
-//
+>([
+  [ContextMenu.SendPageToAgent, sendPageToAgent],
+  [ContextMenu.SendToAgent, sendToAgent],
+  [ContextMenu.SendCaptureToAgent, sendCaptureToAgent],
+  [ContextMenu.SummarizePage, summarizePage],
+]);
 const registerMenu = () => {
+  chrome.contextMenus.create({
+    id: ContextMenu.SummarizePage,
+    title: 'Summarize This Page',
+    contexts: ['all'],
+  });
+  chrome.contextMenus.create({
+    id: 'separator',
+    type: 'separator',
+    contexts: ['all'],
+  });
+  chrome.contextMenus.create({
+    id: ContextMenu.SendPageToAgent,
+    title: 'Send Page to Agent',
+    contexts: ['all'],
+  });
   chrome.contextMenus.create({
     id: ContextMenu.SendToAgent,
-    title: 'Send Selection to Shinkai Chat',
+    title: 'Send Selection to Agent',
     contexts: ['selection'],
   });
+  chrome.contextMenus.create({
+    id: ContextMenu.SendCaptureToAgent,
+    title: 'Send Capture to Agent',
+    contexts: ['all'],
+  });
 };
 
 chrome.runtime.onInstalled.addListener(async (details) => {
   registerMenu();
-
   if (details.reason === 'install') {
     await chrome.tabs.create({
       url: chrome.runtime.getURL('src/components/setup/setup.html'),

diff --git a/libs/shinkai-ui/src/components/message-editor.tsx b/libs/shinkai-ui/src/components/message-editor.tsx
@@ -50,6 +50,10 @@ export const MessageEditor = ({
                 onSubmitRef?.current?.();
                 return this.editor.commands.clearContent();
               },
+              'Mod-Enter': () => {
+                onSubmitRef?.current?.();
+                return this.editor.commands.clearContent();
+              },
               'Shift-Enter': ({ editor }) =>
                 editor.commands.first(({ commands }) => [
                   () => commands.newlineInCode(),

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -160,7 +160,7 @@
     "jsbi": "^4.3.0",
     "jspdf": "^2.5.1",
     "libsodium-wrappers-sumo": "^0.7.13",
-    "lucide-react": "^0.263.1",
+    "lucide-react": "^0.323.0",
     "node-forge": ">=1.0.0",
     "react": "18.2.0",
     "react-dom": "18.2.0",