feat(deepgram): init Deepgram STT

.changeset/friendly-wombats-help.md

@@ -0,0 +1,6 @@
+---
+"@livekit/agents": patch
+"@livekit/agents-plugin-deepgram": minor
+---
+
+add Deepgram text-to-speech plugin

agents/src/utils.ts

@@ -297,6 +297,7 @@ export class AsyncIterableQueue<T> implements AsyncIterableIterator<T> {
   }
 }
 
+/** @internal */
 export class ExpFilter {
   #alpha: number;
   #max?: number;
@@ -337,3 +338,31 @@ export class ExpFilter {
     this.#alpha = alpha;
   }
 }
+
+/** @internal */
+export class AudioEnergyFilter {
+  #cooldownSeconds: number;
+  #cooldown: number;
+
+  constructor(cooldownSeconds = 1) {
+    this.#cooldownSeconds = cooldownSeconds;
+    this.#cooldown = cooldownSeconds;
+  }
+
+  pushFrame(frame: AudioFrame): boolean {
+    const arr = Float32Array.from(frame.data, (x) => x / 32768);
+    const rms = (arr.map((x) => x ** 2).reduce((acc, x) => acc + x) / arr.length) ** 0.5;
+    if (rms > 0.004) {
+      this.#cooldown = this.#cooldownSeconds;
+      return true;
+    }
+
+    const durationSeconds = frame.samplesPerChannel / frame.sampleRate;
+    this.#cooldown -= durationSeconds;
+    if (this.#cooldown > 0) {
+      return true;
+    }
+
+    return false;
+  }
+}

examples/package.json

@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@livekit/agents": "workspace:*",
+    "@livekit/agents-plugin-deepgram": "workspace:*",
     "@livekit/agents-plugin-elevenlabs": "workspace:*",
     "@livekit/agents-plugin-openai": "workspace:*",
     "@livekit/rtc-node": "^0.10.2",

examples/src/stt.ts

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+import { type JobContext, WorkerOptions, cli, defineAgent, stt } from '@livekit/agents';
+import { STT } from '@livekit/agents-plugin-deepgram';
+import type { Track } from '@livekit/rtc-node';
+import { AudioStream, RoomEvent, TrackKind } from '@livekit/rtc-node';
+import { fileURLToPath } from 'node:url';
+
+export default defineAgent({
+  entry: async (ctx: JobContext) => {
+    await ctx.connect();
+    console.log('starting STT example agent');
+
+    const transcribeTrack = async (track: Track) => {
+      const audioStream = new AudioStream(track);
+      const sttStream = new STT({ sampleRate: 48000 }).stream();
+
+      const sendTask = async () => {
+        for await (const event of audioStream) {
+          sttStream.pushFrame(event);
+        }
+      };
+
+      const recvTask = async () => {
+        for await (const event of sttStream) {
+          if (event.type === stt.SpeechEventType.FINAL_TRANSCRIPT) {
+            console.log(event.alternatives[0].text);
+          }
+        }
+      };
+
+      Promise.all([sendTask(), recvTask()]);
+    };
+
+    ctx.room.on(RoomEvent.TrackSubscribed, async (track: Track) => {
+      if (track.kind === TrackKind.KIND_AUDIO) {
+        transcribeTrack(track);
+      }
+    });
+  },
+});
+
+cli.runApp(new WorkerOptions({ agent: fileURLToPath(import.meta.url) }));

plugins/deepgram/api-extractor.json

@@ -0,0 +1,20 @@
+/**
+ * Config file for API Extractor.  For more info, please visit: https://api-extractor.com
+ */
+{
+  "$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
+
+  /**
+   * Optionally specifies another JSON config file that this file extends from.  This provides a way for
+   * standard settings to be shared across multiple projects.
+   *
+   * If the path starts with "./" or "../", the path is resolved relative to the folder of the file that contains
+   * the "extends" field.  Otherwise, the first path segment is interpreted as an NPM package name, and will be
+   * resolved using NodeJS require().
+   *
+   * SUPPORTED TOKENS: none
+   * DEFAULT VALUE: ""
+   */
+  "extends": "../../api-extractor-shared.json",
+  "mainEntryPointFilePath": "./dist/index.d.ts"
+}

plugins/deepgram/package.json

@@ -0,0 +1,27 @@
+{
+  "name": "@livekit/agents-plugin-deepgram",
+  "version": "0.0.0",
+  "description": "Deepgram plugin for LiveKit Agents for Node.js",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "author": "LiveKit",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "clean:build": "pnpm clean && pnpm build",
+    "lint": "eslint -f unix \"src/**/*.{ts,js}\"",
+    "api:check": "api-extractor run --typescript-compiler-folder ../../node_modules/typescript",
+    "api:update": "api-extractor run --local --typescript-compiler-folder ../../node_modules/typescript --verbose"
+  },
+  "devDependencies": {
+    "@microsoft/api-extractor": "^7.35.0",
+    "@types/ws": "^8.5.10",
+    "typescript": "^5.0.0"
+  },
+  "dependencies": {
+    "@livekit/agents": "workspace:*",
+    "@livekit/rtc-node": "^0.10.2",
+    "ws": "^8.16.0"
+  }
+}

plugins/deepgram/src/index.ts

@@ -0,0 +1,5 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export * from './stt.js';

plugins/deepgram/src/models.ts

@@ -0,0 +1,70 @@
+// SPDX-FileCopyrightText: 2024 LiveKit, Inc.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+export type STTModels =
+  | 'nova-general'
+  | 'nova-phonecall'
+  | 'nova-meeting'
+  | 'nova-2-general'
+  | 'nova-2-meeting'
+  | 'nova-2-phonecall'
+  | 'nova-2-finance'
+  | 'nova-2-conversationalai'
+  | 'nova-2-voicemail'
+  | 'nova-2-video'
+  | 'nova-2-medical'
+  | 'nova-2-drivethru'
+  | 'nova-2-automotive'
+  | 'enhanced-general'
+  | 'enhanced-meeting'
+  | 'enhanced-phonecall'
+  | 'enhanced-finance'
+  | 'base'
+  | 'meeting'
+  | 'phonecall'
+  | 'finance'
+  | 'conversationalai'
+  | 'voicemail'
+  | 'video'
+  | 'whisper-tiny'
+  | 'whisper-base'
+  | 'whisper-small'
+  | 'whisper-medium'
+  | 'whisper-large';
+
+export type STTLanguages =
+  | 'da'
+  | 'de'
+  | 'en'
+  | 'en-AU'
+  | 'en-GB'
+  | 'en-IN'
+  | 'en-NZ'
+  | 'en-US'
+  | 'es'
+  | 'es-419'
+  | 'es-LATAM'
+  | 'fr'
+  | 'fr-CA'
+  | 'hi'
+  | 'hi-Latn'
+  | 'id'
+  | 'it'
+  | 'ja'
+  | 'ko'
+  | 'nl'
+  | 'no'
+  | 'pl'
+  | 'pt'
+  | 'pt-BR'
+  | 'ru'
+  | 'sv'
+  | 'ta'
+  | 'taq'
+  | 'th'
+  | 'tr'
+  | 'uk'
+  | 'zh'
+  | 'zh-CN'
+  | 'zh-TW';