- feature: added support for python code execution (#461)

* first try

* - wip: working version of python execution

* failed attempt

* - fix: make lib works

* - improvements: working version with plotly, error and result render

* - fix: parse plotly data

* - refactor: added web worker implementation

* - refactor: render figures from pandas and plotly

* minor ui improvements

* - refactor: removed unused code

* - fix: added i18n

* - fix: added missing i18n to default file

* - fix: removed unused file

* - fix: added patch to execute network requests

* - fix: csp set to null again, should be fixed on upcoming prs

* - fix: workaround for worker stuffs coming from shinkai-ui

* - cicd: version bump

* - test: disable bundleMediaFramework for linux

---------

Co-authored-by: Alfredo Gallardo <[email protected]>
Co-authored-by: paulclindo <[email protected]>

apps/shinkai-desktop/src-tauri/Cargo.toml

@@ -37,6 +37,8 @@ base64 = "0.22.1"
 time = "^0.3.36"
 listeners = "0.2"
 log = "0.4"
+anyhow = "1.0.89"
+uuid = "1.10.0"
 
 tauri-plugin-global-shortcut = "2.0.0-rc.2"
 tauri-plugin-shell = "2.0.0-rc.3"

apps/shinkai-desktop/src-tauri/src/main.rs

@@ -84,7 +84,7 @@ fn main() {
             shinkai_node_get_ollama_api_url,
             shinkai_node_get_default_model,
             hardware_get_summary,
-            galxe_generate_proof
+            galxe_generate_proof,
         ])
         .setup(|app| {
             let app_resource_dir = app.path().resource_dir()?;

apps/shinkai-desktop/src-tauri/src/windows/mod.rs

@@ -1,4 +1,4 @@
-use tauri::{AppHandle, WebviewWindowBuilder, Manager};
+use tauri::{AppHandle, Manager, WebviewWindowBuilder};
 
 #[derive(Debug, Clone, Copy)]
 pub enum Window {
@@ -17,7 +17,6 @@ impl Window {
     }
 }
 
-
 pub fn show_or_recreate_window(app_handle: AppHandle, window_name: Window) {
     let label = window_name.as_str();
     if let Some(window) = app_handle.get_webview_window(label) {
@@ -27,13 +26,20 @@ pub fn show_or_recreate_window(app_handle: AppHandle, window_name: Window) {
         let _ = window.set_focus();
     } else {
         log::info!("window {} not found, recreating...", label);
-        let window_config = app_handle.config().app.windows.iter().find(|w| w.label == label).unwrap().clone();
+        let window_config = app_handle
+            .config()
+            .app
+            .windows
+            .iter()
+            .find(|w| w.label == label)
+            .unwrap()
+            .clone();
         match WebviewWindowBuilder::from_config(&app_handle, &window_config) {
             Ok(builder) => {
                 if let Err(e) = builder.build() {
                     log::error!("failed to recreate window: {}", e);
                 }
-            },
+            }
             Err(e) => {
                 log::error!("failed to recreate window from config: {}", e);
             }

apps/shinkai-desktop/src-tauri/tauri.linux.conf.json

@@ -3,7 +3,7 @@
     "resources": ["external-binaries/shinkai-node/libpdfium.so"],
     "linux": {
       "appimage": {
-        "bundleMediaFramework": true
+        "bundleMediaFramework": false
       }
     }
   }

apps/shinkai-desktop/src/App.tsx

@@ -1,6 +1,7 @@
 import { I18nProvider } from '@shinkai_network/shinkai-i18n';
 import { QueryProvider } from '@shinkai_network/shinkai-node-state';
 import { Toaster } from '@shinkai_network/shinkai-ui';
+import { useEffect } from 'react';
 import { ErrorBoundary } from 'react-error-boundary';
 import { BrowserRouter as Router } from 'react-router-dom';
 

apps/shinkai-desktop/src/components/error-boundary.tsx

@@ -1,13 +1,22 @@
+import { Button } from "@shinkai_network/shinkai-ui";
+
 const FullPageErrorFallback = ({ error }: { error: Error }) => {
   return (
     <div
       className="flex h-screen flex-col items-center justify-center px-8 text-red-400"
       role="alert"
     >
       <p>Something went wrong. Try refreshing the app.</p>
-      <pre className="whitespace-pre-wrap text-balance break-all text-center">
+      <pre className="whitespace-pre-wrap text-balance break-all text-center mb-4">
         {error.message}
       </pre>
+      <Button
+        onClick={() => window.location.reload()}
+        size="sm"
+        variant="secondary"
+      >
+        Refresh
+      </Button>
     </div>
   );
 };

apps/shinkai-desktop/src/windows/python-runner/index.html

@@ -0,0 +1,14 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Python Runner</title>
+  </head>
+
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/windows/python-runner/main.ts"></script>
+  </body>
+</html>

apps/shinkai-desktop/src/windows/python-runner/main.ts

@@ -0,0 +1,173 @@
+import { emit, Event, listen } from '@tauri-apps/api/event';
+import { loadPyodide, PyodideInterface } from 'pyodide';
+
+const INDEX_URL = 'https://cdn.jsdelivr.net/pyodide/v0.26.2/full/';
+
+let pyodide: PyodideInterface;
+const stdout: string[] = [];
+const stderr: string[] = [];
+
+const getWrappedCode = (code: string): string => {
+  const wrappedCode = `
+import sys
+from io import StringIO
+import pandas as pd
+import plotly.graph_objects as go
+import plotly.express as px
+import plotly.io as pio
+import json
+
+# Redirect stdout to capture prints
+old_stdout = sys.stdout
+sys.stdout = mystdout = StringIO()
+
+# Function to capture DataFrame display as HTML
+def capture_df_display(df):
+    return df.to_html()
+
+output = None
+outputError = None
+fig_json = None
+local_scope = {}
+try:
+    exec("""${code}""", globals(), local_scope)
+    if 'output' in local_scope and isinstance(local_scope['output'], pd.DataFrame):
+        output = capture_df_display(local_scope['output'])
+    else:
+        # Capture the last variable in the scope
+        last_var = list(local_scope.values())[-1] if local_scope else None
+        if isinstance(last_var, pd.DataFrame):
+            output = capture_df_display(last_var)
+        elif isinstance(last_var, (str, int, float, list, dict)):
+            output = json.dumps(last_var)
+        else:
+            output = mystdout.getvalue().strip()
+    for var_name, var_value in local_scope.items():
+        if isinstance(var_value, go.Figure):
+            fig_json = pio.to_json(var_value)
+            break
+except Exception as e:
+    outputError = str(e)
+
+# Reset stdout
+sys.stdout = old_stdout
+
+(output, outputError, fig_json)
+    `;
+  return wrappedCode;
+};
+const runPythonCode = async (
+  code: string,
+): Promise<{ type: string; data: string }> => {
+  const [output, outputError, figJson] = await pyodide.runPythonAsync(code);
+  if (outputError) {
+    throw new Error(outputError);
+  }
+  if (figJson) {
+    return { type: 'plotly', data: figJson };
+  }
+  return { type: 'string', data: output };
+};
+
+listen('run', async (event: Event<string>) => {
+  if (!pyodide) {
+    const error = new Error('pyodide is not initialized');
+    emit('execution-error', error);
+    throw error;
+  }
+  try {
+    const startTime = performance.now();
+
+    console.log('Starting code execution');
+    const code = getWrappedCode(event.payload);
+    console.log('Code wrapped', performance.now() - startTime, 'ms');
+
+    console.log('Loading packages from imports');
+    const loadPackagesStart = performance.now();
+    await pyodide.loadPackagesFromImports(code, {
+      messageCallback: (message) => {
+        console.log('Load package message:', message);
+      },
+      errorCallback: (message) => {
+        console.log('Load package error:', message);
+      },
+    });
+    console.log('Packages loaded', performance.now() - loadPackagesStart, 'ms');
+
+    console.log('Running Python code');
+    const runCodeStart = performance.now();
+    const result = await runPythonCode(code);
+    console.log('Python code executed', performance.now() - runCodeStart, 'ms');
+
+    console.log('Pyodide run result:', result);
+
+    const totalTime = performance.now() - startTime;
+    console.log('Total execution time:', totalTime, 'ms');
+
+    emit('execution-result', {
+      state: 'success',
+      stdout,
+      stderr,
+      payload: result,
+    });
+  } catch (e) {
+    console.log('Pyodide run error:', e);
+    emit('execution-result', {
+      state: 'error',
+      stdout,
+      stderr,
+      payload: String(e),
+    });
+  }
+});
+
+const main = async () => {
+  try {
+    console.log('loading pyodide');
+    const startLoadPyodide = performance.now();
+    pyodide = await loadPyodide({
+      indexURL: INDEX_URL,
+      stdout: (message) => {
+        console.log('python stdout', message);
+        stdout.push(message);
+      },
+      stderr: (message) => {
+        console.log('python stderr', message);
+        stderr.push(message);
+      },
+      fullStdLib: true,
+    });
+    const endLoadPyodide = performance.now();
+    console.log(`Pyodide loaded in ${endLoadPyodide - startLoadPyodide} ms`);
+
+    const startLoadPackages = performance.now();
+    await pyodide.loadPackage(['micropip', 'pandas', 'numpy', 'matplotlib']);
+    const endLoadPackages = performance.now();
+    console.log(`Packages loaded in ${endLoadPackages - startLoadPackages} ms`);
+
+    const startMicropip = performance.now();
+    const micropip = pyodide.pyimport('micropip');
+    const endMicropip = performance.now();
+    console.log(`Micropip imported in ${endMicropip - startMicropip} ms`);
+
+    const startPlotly = performance.now();
+    await micropip.install('plotly');
+    const endPlotly = performance.now();
+    console.log(`Plotly installed in ${endPlotly - startPlotly} ms`);
+
+    const startSeaborn = performance.now();
+    await micropip.install('seaborn');
+    const endSeaborn = performance.now();
+    console.log(`Seaborn installed in ${endSeaborn - startSeaborn} ms`);
+
+    console.log('pyodide loaded successfully');
+    const totalTime = performance.now() - startLoadPyodide;
+    console.log(`Total loading time: ${totalTime} ms`);
+    emit('ready');
+  } catch (e) {
+    console.log('pyodide load error', e);
+    emit('loading-error');
+    return;
+  }
+};
+main();

apps/shinkai-desktop/vite.config.ts

@@ -63,4 +63,7 @@ export default defineConfig(() => ({
       },
     },
   },
+  worker: {
+    format: 'es' as const
+  },
 }));

apps/shinkai-visor/vite.config.ts

@@ -59,4 +59,7 @@ export default defineConfig({
       },
     },
   },
+  worker: {
+    format: 'es' as const,
+  }
 });

libs/shinkai-i18n/locales/en-US.json

@@ -537,5 +537,12 @@
       "label": "Message Received",
       "description": "You have received a response from {{inboxName}}"
     }
+  },
+  "codeRunner": {
+    "errorOccurred": "Error Occurred",
+    "stderr": "Standard Error",
+    "stdout": "Standard Output",
+    "output": "Output",
+    "executeCode": "Execute Code"
   }
-}
+}

libs/shinkai-i18n/locales/es-ES.json

@@ -539,5 +539,12 @@
   },
   "workflowPlayground": {
     "label": "Área de Juego de Flujo de Trabajo"
+  },
+  "codeRunner": {
+    "errorOccurred": "Ocurrió un Error",
+    "stderr": "Salida de Error",
+    "stdout": "Salida Estándar",
+    "output": "Resultado",
+    "executeCode": "Ejecutar Código"
   }
 }

libs/shinkai-i18n/locales/id-ID.json

@@ -537,5 +537,12 @@
   },
   "workflowPlayground": {
     "label": "Taman Bermain Alur Kerja"
+  },
+  "codeRunner": {
+    "errorOccurred": "Terjadi Kesalahan",
+    "stderr": "Kesalahan Standar",
+    "stdout": "Keluaran Standar",
+    "output": "Hasil",
+    "executeCode": "Jalankan Kode"
   }
 }

libs/shinkai-i18n/locales/ja-JP.json

@@ -537,5 +537,12 @@
   },
   "workflowPlayground": {
     "label": "ワークフロープレイグラウンド"
+  },
+  "codeRunner": {
+    "errorOccurred": "エラーが発生しました",
+    "stderr": "標準エラー",
+    "stdout": "標準出力",
+    "output": "出力",
+    "executeCode": "コードを実行"
   }
 }