From 6e0cb7b0853b6637998a4ecf003d655e2a449ebf Mon Sep 17 00:00:00 2001 From: Nicolas Molina Monroy Date: Mon, 15 Apr 2024 03:10:46 -0400 Subject: [PATCH] Add CLOUDFLARE_AI_GATEWAY_URL environment variable and update generateAudio function in openai.service.ts --- apps/api/src/bindings.ts | 1 + apps/api/src/index.ts | 48 +++++++++++- apps/api/src/routes/createQuestion.ts | 2 +- apps/api/src/routes/createTranscript.ts | 8 +- apps/api/src/services/openai.service.ts | 12 +-- apps/webapp/src/app/app.routes.ts | 4 + .../modal-recording.component.ts | 8 +- .../src/app/pages/demo/demo.component.html | 16 ++++ .../src/app/pages/demo/demo.component.ts | 75 +++++++++++++++++++ .../pages/simulator/simulator.component.ts | 2 +- apps/webapp/src/app/services/api.service.ts | 5 +- 11 files changed, 159 insertions(+), 22 deletions(-) create mode 100644 apps/webapp/src/app/pages/demo/demo.component.html create mode 100644 apps/webapp/src/app/pages/demo/demo.component.ts diff --git a/apps/api/src/bindings.ts b/apps/api/src/bindings.ts index 38d2dc8..f827026 100644 --- a/apps/api/src/bindings.ts +++ b/apps/api/src/bindings.ts @@ -2,6 +2,7 @@ export type Bindings = { DB: D1Database; CLOUDFLARE_ACCOUNT_ID: string; CLOUDFLARE_API_TOKEN: string; + CLOUDFLARE_AI_GATEWAY_URL: string; OPENAI_API_KEY: string; AI: any; BUCKET: R2Bucket; diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 589bcd1..f4cd2a2 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -9,8 +9,9 @@ import createSimulation from '@src/routes/createSimulation'; import createQuestion from '@src/routes/createQuestion'; import createFeedback from '@src/routes/createFeedback'; import createTranscript from '@src/routes/createTranscript'; +import { App } from "@src/types"; -const app = new OpenAPIHono(); +const app = new OpenAPIHono(); app.use("*", cors()); app.use('*', prettyJSON()); app.notFound((c) => c.json({ message: 'Not Found', ok: false }, 404)); @@ -22,6 +23,51 @@ app.route('/api/v1/questions', createQuestion); app.route('/api/v1/feedback', createFeedback); app.route('/api/v1/transcript', createTranscript); +/* +import { Ai } from '@cloudflare/ai'; + +app.post('/api/v1/demo1', async (c) => { + const ai = new Ai(c.env.AI); + const arrayBuffer = await c.req.arrayBuffer(); + const audio = [...new Uint8Array(arrayBuffer)]; + const { text } = await ai.run("@cf/openai/whisper", {audio }); + return c.json({text}); +}); + +app.post('/api/v1/demo2', async (c) => { + const ai = new Ai(c.env.AI); + const data = await c.req.formData(); + const file = data.get('file') as unknown as File; + const arrayBuffer = await file.arrayBuffer(); + const audio = [...new Uint8Array(arrayBuffer)]; + const { text } = await ai.run("@cf/openai/whisper", { audio }); + return c.json({text}); +}); + +app.post('/api/v1/demo3', async (c) => { + const data = await c.req.formData(); + const file = data.get('file') as unknown as File; + const text = await generateTranscription(file, c.env.OPENAI_API_KEY); + return c.json({text}); +}); + +app.post('/api/v1/demo4', async (c) => { + const data = await c.req.formData(); + const file = data.get('file') as unknown as File; + const arrayBuffer = await file.arrayBuffer(); + const response = await fetch('https://gateway.ai.cloudflare.com/v1/b2bb1719bede14df8732870a3974b263/gateway/workers-ai/@cf/openai/whisper', { + method: 'POST', + headers: { + 'Authorization': `Bearer ${c.env.CLOUDFLARE_API_TOKEN}`, + 'Content-Type': 'application/octet-stream', + }, + body: arrayBuffer, + }); + const result = await response.json(); + return c.json({result}); +}); +*/ + app.get("/", swaggerUI({ url: "/docs" })); app.doc("/docs", { info: { diff --git a/apps/api/src/routes/createQuestion.ts b/apps/api/src/routes/createQuestion.ts index e908f7e..68e59d8 100644 --- a/apps/api/src/routes/createQuestion.ts +++ b/apps/api/src/routes/createQuestion.ts @@ -45,7 +45,7 @@ app.openapi(route, async (c) => { cloudflareApiToken: c.env.CLOUDFLARE_API_TOKEN, }, c.env.DB); - const filename = await generateAudio(response, c.env.OPENAI_API_KEY, c.env.BUCKET); + const filename = await generateAudio(response, c.env.OPENAI_API_KEY, c.env.CLOUDFLARE_AI_GATEWAY_URL, c.env.BUCKET); return c.json({ id: `${Date.now()}`, diff --git a/apps/api/src/routes/createTranscript.ts b/apps/api/src/routes/createTranscript.ts index 4ae78da..65dea0f 100644 --- a/apps/api/src/routes/createTranscript.ts +++ b/apps/api/src/routes/createTranscript.ts @@ -1,6 +1,6 @@ import { OpenAPIHono, createRoute } from "@hono/zod-openapi"; import { MessageSchema } from '@src/dtos/message.dto'; -import { generateTranscription } from '@src/services/whisper.service'; +import { generateTranscription } from '@src/services/openai.service'; import { App } from "@src/types"; const app = new OpenAPIHono(); @@ -23,9 +23,9 @@ const route = createRoute({ }); app.openapi(route, async (c) => { - const body = await c.req.parseBody(); - const file = body.file as File; - const answer = await generateTranscription(file, c.env.AI); + const data = await c.req.formData(); + const file = data.get('file') as unknown as File; + const answer = await generateTranscription(file, c.env.OPENAI_API_KEY, c.env.CLOUDFLARE_AI_GATEWAY_URL); return c.json({ id: `${Date.now()}`, diff --git a/apps/api/src/services/openai.service.ts b/apps/api/src/services/openai.service.ts index 8f4e29d..d223ce9 100644 --- a/apps/api/src/services/openai.service.ts +++ b/apps/api/src/services/openai.service.ts @@ -1,9 +1,7 @@ import OpenAI from "openai"; -export const generateTranscription = async (file: File, key: string) => { - const openai = new OpenAI({ - apiKey: key, - }); +export const generateTranscription = async (file: File, apiKey: string, baseURL: string) => { + const openai = new OpenAI({ apiKey, baseURL: `${baseURL}/openai`}); console.log('name', file.name); console.log('type', file.type); const transcription = await openai.audio.transcriptions.create({ @@ -14,10 +12,8 @@ export const generateTranscription = async (file: File, key: string) => { return transcription.text; } -export const generateAudio = async (input: string, key: string, bucket: R2Bucket) => { - const openai = new OpenAI({ - apiKey: key, - }); +export const generateAudio = async (input: string, apiKey: string, baseURL: string, bucket: R2Bucket) => { + const openai = new OpenAI({ apiKey, baseURL: `${baseURL}/openai`}); const file = await openai.audio.speech.create({ model: "tts-1", voice: "alloy", diff --git a/apps/webapp/src/app/app.routes.ts b/apps/webapp/src/app/app.routes.ts index fed2429..04ae1a7 100644 --- a/apps/webapp/src/app/app.routes.ts +++ b/apps/webapp/src/app/app.routes.ts @@ -13,6 +13,10 @@ export const routes: Routes = [ path: 'simulator/:id', loadComponent: () => import('./pages/simulator/simulator.component') }, + { + path: 'demo', + loadComponent: () => import('./pages/demo/demo.component') + }, { path: '**', redirectTo: '' diff --git a/apps/webapp/src/app/components/modal-recording/modal-recording.component.ts b/apps/webapp/src/app/components/modal-recording/modal-recording.component.ts index 1d0f08d..306bec1 100644 --- a/apps/webapp/src/app/components/modal-recording/modal-recording.component.ts +++ b/apps/webapp/src/app/components/modal-recording/modal-recording.component.ts @@ -31,7 +31,7 @@ export class ModalRecordingComponent implements AfterViewInit { @ViewChild('videoRta', { static: false }) videoRta!: ElementRef; private mediaRecorder!: MediaRecorder; status = signal<'init' | 'recording' | 'success' | 'processing' | 'streaming'>('init'); - file = signal(null); + file = signal(null); intervalId = signal(null); duration = signal(0); formatDuration = computed(() => { @@ -77,11 +77,9 @@ export class ModalRecordingComponent implements AfterViewInit { previewVideo() { const chunks = this.recordedChunks(); const blob = new Blob(chunks, { type: 'video/webm' }); - const filename = `${Date.now()}.webm`; - const file = new File([blob], filename, { type: blob.type, lastModified: Date.now() }); - const url = URL.createObjectURL(file); + const url = URL.createObjectURL(blob); this.status.set('success'); - this.file.set(file); + this.file.set(blob); this.videoRta.nativeElement.src = url; this.cdRef.detectChanges(); } diff --git a/apps/webapp/src/app/pages/demo/demo.component.html b/apps/webapp/src/app/pages/demo/demo.component.html new file mode 100644 index 0000000..de8c022 --- /dev/null +++ b/apps/webapp/src/app/pages/demo/demo.component.html @@ -0,0 +1,16 @@ +
+

+ +

+

+ +

+ @if(url()) { +

+ +

+ } + +
diff --git a/apps/webapp/src/app/pages/demo/demo.component.ts b/apps/webapp/src/app/pages/demo/demo.component.ts new file mode 100644 index 0000000..e4b64b3 --- /dev/null +++ b/apps/webapp/src/app/pages/demo/demo.component.ts @@ -0,0 +1,75 @@ +import { Component, inject, signal } from '@angular/core'; +import { HttpClient } from '@angular/common/http'; +import { environment } from '@env/environment'; + +@Component({ + selector: 'app-demo', + standalone: true, + imports: [], + templateUrl: './demo.component.html', + styles: `` +}) +export default class DemoComponent { + recordedChunks = signal([]); + url = signal(null); + private mediaRecorder!: MediaRecorder; + + private http = inject(HttpClient); + + async startRecording() { + const stream = await navigator.mediaDevices.getUserMedia({ + video: true, + audio: true, + }); + this.mediaRecorder = new MediaRecorder(stream, { mimeType: 'video/webm;codecs=vp9,opus' }); + this.mediaRecorder.ondataavailable = (event) => { + if (event.data.size > 0) { + this.recordedChunks.update((chunks) => [...chunks, event.data]); + } + }; + this.mediaRecorder.start(); + } + + stopRecording() { + this.mediaRecorder.stop(); + setTimeout(() => { + this.processRecording(); + }, 300); + } + + processRecording() { + const chunks = this.recordedChunks(); + console.log(chunks); + const blob = new Blob(chunks, { type: 'video/webm' }); + const url = URL.createObjectURL(blob); + this.url.set(url); + this.requestOpenAI(blob).subscribe((rta) => { + console.log(rta); + this.recordedChunks.set([]); + this.url.set(null); + }); + } + + requestBlob(file: Blob) { + return this.http.post(`${environment.apiUrl}/demo1`, file); + } + + requestFormData(file: Blob) { + const formData = new FormData(); + formData.append('file', file, 'file.webm'); + return this.http.post(`${environment.apiUrl}/demo2`, formData); + } + + requestOpenAI(file: Blob) { + const formData = new FormData(); + formData.append('file', file, 'file.webm'); + return this.http.post(`${environment.apiUrl}/demo3`, formData); + } + + requestGateway(file: Blob) { + const formData = new FormData(); + formData.append('file', file, 'file.webm'); + return this.http.post(`${environment.apiUrl}/demo4`, formData); + } + +} diff --git a/apps/webapp/src/app/pages/simulator/simulator.component.ts b/apps/webapp/src/app/pages/simulator/simulator.component.ts index 1352148..2f2f94b 100644 --- a/apps/webapp/src/app/pages/simulator/simulator.component.ts +++ b/apps/webapp/src/app/pages/simulator/simulator.component.ts @@ -63,7 +63,7 @@ export default class SimulatorComponent implements OnInit { }); } - createTranscript(file: File, question: string) { + createTranscript(file: Blob, question: string) { this.mode.set('loading'); this.apiService.createTranscript(file).subscribe({ next: (newMessage) => { diff --git a/apps/webapp/src/app/services/api.service.ts b/apps/webapp/src/app/services/api.service.ts index d8ed37c..2bd40a4 100644 --- a/apps/webapp/src/app/services/api.service.ts +++ b/apps/webapp/src/app/services/api.service.ts @@ -30,9 +30,10 @@ export class ApiService { }); } - createTranscript(file: File) { + createTranscript(file: Blob) { const formData = new FormData(); - formData.append('file', file); + const filename = `file-${Date.now()}.webm`; + formData.append('file', file, filename); return this.http.post(`${environment.apiUrl}/transcript`, formData) .pipe( map((response) => ({