diff --git a/components/speak_ai/actions/find-media/find-media.mjs b/components/speak_ai/actions/find-media/find-media.mjs new file mode 100644 index 0000000000000..66684f3221f8a --- /dev/null +++ b/components/speak_ai/actions/find-media/find-media.mjs @@ -0,0 +1,52 @@ +import app from "../../speak_ai.app.mjs"; + +export default { + key: "speak_ai-find-media", + name: "Find Media", + description: "Find an existing Speak AI media item by its Media ID. [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: true, + }, + type: "action", + props: { + app, + folderId: { + propDefinition: [ + app, + "folderId", + ], + }, + mediaId: { + propDefinition: [ + app, + "mediaId", + ({ folderId }) => ({ + folderId, + }), + ], + }, + }, + async run({ $ }) { + const { + app, + mediaId, + } = this; + + const results = await app.getInsights({ + $, + params: { + mediaId, + pageSize: 1, + }, + }); + const media = app.firstResult(results); + + $.export("$summary", media + ? `Found media \`${mediaId}\`.` + : `No media found for \`${mediaId}\`.`); + return media; + }, +}; diff --git a/components/speak_ai/actions/run-ai-chat/run-ai-chat.mjs b/components/speak_ai/actions/run-ai-chat/run-ai-chat.mjs new file mode 100644 index 0000000000000..0ecd8b4fd9437 --- /dev/null +++ b/components/speak_ai/actions/run-ai-chat/run-ai-chat.mjs @@ -0,0 +1,75 @@ +import app from "../../speak_ai.app.mjs"; + +export default { + key: "speak_ai-run-ai-chat", + name: "Run AI Chat", + description: "Run a Speak AI Chat against a folder and/or specific media. [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + annotations: { + destructiveHint: false, + openWorldHint: true, + readOnlyHint: false, + }, + type: "action", + props: { + app, + prompt: { + propDefinition: [ + app, + "prompt", + ], + }, + folderId: { + propDefinition: [ + app, + "folderId", + ], + }, + assistantType: { + type: "string", + label: "Assistant Type", + description: "The assistant type to use for the prompt (e.g. `researcher`, `marketer`, `sales`, `general`, `recruiter`)", + optional: true, + default: "general", + }, + assistantTemplateId: { + type: "string", + label: "Assistant Template ID", + description: "Optional custom assistant template to apply", + optional: true, + }, + mediaIds: { + propDefinition: [ + app, + "mediaIds", + ], + }, + }, + async run({ $ }) { + const { + app, + prompt, + folderId, + assistantType, + assistantTemplateId, + mediaIds, + } = this; + + const response = await app.runPrompt({ + $, + data: { + folderId, + prompt, + assistantType, + assistantTemplateId, + mediaIds: mediaIds || [], + }, + }); + + const promptId = response?.data?.promptId; + $.export("$summary", promptId + ? `Successfully ran AI Chat \`${promptId}\`.` + : "Successfully ran AI Chat."); + return response; + }, +}; \ No newline at end of file diff --git a/components/speak_ai/package.json b/components/speak_ai/package.json index 1d615a5c9db33..989dd4fda09e5 100644 --- a/components/speak_ai/package.json +++ b/components/speak_ai/package.json @@ -1,6 +1,6 @@ { "name": "@pipedream/speak_ai", - "version": "0.2.0", + "version": "0.3.0", "description": "Pipedream Speak AI Components", "main": "speak_ai.app.mjs", "keywords": [ diff --git a/components/speak_ai/sources/common/base.mjs b/components/speak_ai/sources/common/base.mjs new file mode 100644 index 0000000000000..c59e704624694 --- /dev/null +++ b/components/speak_ai/sources/common/base.mjs @@ -0,0 +1,65 @@ +import { ConfigurationError } from "@pipedream/platform"; +import app from "../../speak_ai.app.mjs"; +import constants from "../../common/constants.mjs"; + +export default { + props: { + app, + db: "$.service.db", + http: "$.interface.http", + }, + hooks: { + async activate() { + const { data } = await this.app.subscribeWebhook({ + data: { + callbackUrl: this.http.endpoint, + events: this.getEvents(), + }, + }); + this.setWebhookId(data.webhookId); + }, + async deactivate() { + const webhookId = this.getWebhookId(); + if (webhookId) { + await this.app.unsubscribeWebhook({ + webhookId, + }); + } + }, + }, + methods: { + setWebhookId(value) { + this.db.set(constants.WEBHOOK_ID, value); + }, + getWebhookId() { + return this.db.get(constants.WEBHOOK_ID); + }, + getEvents() { + throw new ConfigurationError("getEvents is not implemented"); + }, + getSummary(resource) { + return `New Speak AI event: ${resource.eventType || "delivery"}`; + }, + async hydrate(resource) { + return resource; + }, + generateMeta(resource, data) { + return { + id: resource.deliveryId || resource.mediaId || resource.messageId || resource.promptId, + summary: this.getSummary(resource, data), + ts: Date.parse(resource.createdAt) || Date.now(), + }; + }, + async processResource(resource) { + const data = await this.hydrate(resource); + this.$emit(data, this.generateMeta(resource, data)); + }, + }, + async run({ body }) { + if (!body || typeof body !== "object" || Array.isArray(body)) { + console.log("Skipping event: webhook body is missing or not a resource object."); + return; + } + await this.processResource(body); + }, +}; diff --git a/components/speak_ai/sources/new-ai-chat/new-ai-chat.mjs b/components/speak_ai/sources/new-ai-chat/new-ai-chat.mjs new file mode 100644 index 0000000000000..50ac311e3186a --- /dev/null +++ b/components/speak_ai/sources/new-ai-chat/new-ai-chat.mjs @@ -0,0 +1,47 @@ +import common from "../common/base.mjs"; +import events from "../common/events.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "speak_ai-new-ai-chat", + name: "New AI Chat Response (Instant)", + description: "Emit new event when a Speak AI Chat response is ready (`chat.status`). [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getEvents() { + return [ + events.CHAT_STATUS, + ]; + }, + getSummary(resource) { + return `New AI Chat response ${resource.messageId || resource.promptId || ""}`.trim(); + }, + async hydrate(resource) { + const res = await this.app.getPromptsHistory({ + headers: { + Accept: "application/json", + }, + params: { + pageSize: 100, + }, + }); + const history = res?.data?.history || []; + const match = history.find((item) => + (resource.messageId && item.messageId === resource.messageId) + || (resource.promptId && item.promptId === resource.promptId)); + return match || { + promptId: resource.promptId, + messageId: resource.messageId, + prompt: resource.prompt, + answer: resource.answer, + mediaIds: resource.mediaIds, + folderId: resource.folderId, + }; + }, + }, + sampleEmit, +}; diff --git a/components/speak_ai/sources/new-ai-chat/test-event.mjs b/components/speak_ai/sources/new-ai-chat/test-event.mjs new file mode 100644 index 0000000000000..43a227f28969d --- /dev/null +++ b/components/speak_ai/sources/new-ai-chat/test-event.mjs @@ -0,0 +1,39 @@ +export default { + "id": "64c2cfc863d1160cd6c2262a", + "promptId": "64c2cfc863d1160cd6c2262a", + "messageId": "45df1bc7b1b2", + "prompt": "What challenges is the team at Speak AI having with software sales?", + "answer": "From the text, it is mentioned that the team at Speak AI is facing several challenges with software sales, including customer segmentation, competition, and limited resources.", + "createdAt": "2023-07-27T20:12:56.645Z", + "assistantType": "General", + "mediaName": "Speak AI Office Hours", + "mediaIds": [], + "folderId": "27c6e4a90e32", + "state": "completed", + "link": "https://app.speakai.co/media/list?dialog=chat&tab=history", + "references": [ + { + "companyId": "5e21c8dd2d77242c64214816", + "createdAt": "2021-08-06T17:39:24.934Z", + "folderId": "27c6e4a90e32", + "mediaId": "a59b93b67685", + "name": "Office Hours 28", + "tags": [ + "Office Hours" + ], + "type": "video", + "userId": "5d03a9d5d4bca272e9c8cf89", + "sentences": [ + { + "speaker": "Vatsal Shah", + "startTime": "0:04:41.38", + "endTime": "0:05:26.46", + "text": "Working on the engineering side building the product and moving to the customer segmentation side.", + "link": "", + "score": 0.84037739 + } + ], + "score": 0.84037739 + } + ] +}; diff --git a/components/speak_ai/sources/new-captions/new-captions.mjs b/components/speak_ai/sources/new-captions/new-captions.mjs new file mode 100644 index 0000000000000..d76b02a1aa7e1 --- /dev/null +++ b/components/speak_ai/sources/new-captions/new-captions.mjs @@ -0,0 +1,51 @@ +import common from "../common/base.mjs"; +import events from "../common/events.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "speak_ai-new-captions", + name: "New Captions (Instant)", + description: "Emit new event with caption files (SRT or VTT) when Speak AI finishes analyzing a media file (`media.analyzed`, `media.reanalyzed`). [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + type: "source", + dedupe: "unique", + props: { + ...common.props, + fileType: { + type: "string", + label: "Caption File Type", + description: "The caption format to fetch", + options: [ + "SRT", + "VTT", + ], + default: "SRT", + }, + }, + methods: { + ...common.methods, + getEvents() { + return [ + events.MEDIA_ANALYZED, + events.MEDIA_REANALYZED, + ]; + }, + getSummary(resource) { + return `New ${this.fileType} captions for media ${resource.mediaId}`; + }, + async hydrate(resource) { + const results = await this.app.getExport({ + headers: { + Accept: "application/json", + }, + params: { + mediaId: resource.mediaId, + fileType: this.fileType.toLowerCase(), + }, + }); + return this.app.firstResult(results, resource); + }, + }, + sampleEmit, +}; diff --git a/components/speak_ai/sources/new-captions/test-event.mjs b/components/speak_ai/sources/new-captions/test-event.mjs new file mode 100644 index 0000000000000..656fff439642c --- /dev/null +++ b/components/speak_ai/sources/new-captions/test-event.mjs @@ -0,0 +1,8 @@ +export default { + "id": "4f42d4179f68", + "mediaId": "4f42d4179f68", + "name": "GMT20211029-100117_Recording_1920x1080", + "description": "testing 123", + "durationInSecond": "185", + "caption": "1\n00:00:11,780 --> 00:00:13,750\nHi, that's all I just wanted to\n\n2\n00:00:15,180 --> 00:00:20,050\nwrite you a message, but it's taking\ntoo long so I think it's maybe it's\njust easier if I.\n" +}; diff --git a/components/speak_ai/sources/new-recording/new-recording.mjs b/components/speak_ai/sources/new-recording/new-recording.mjs new file mode 100644 index 0000000000000..b402cfaef9ab6 --- /dev/null +++ b/components/speak_ai/sources/new-recording/new-recording.mjs @@ -0,0 +1,36 @@ +import common from "../common/base.mjs"; +import events from "../common/events.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "speak_ai-new-recording", + name: "New Recording Captured (Instant)", + description: "Emit new event when a recording is captured through a Speak AI embed recorder (`embed_recorder.recording_received`). [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getEvents() { + return [ + events.EMBED_RECORDER_RECORDING_RECEIVED, + ]; + }, + getSummary(resource) { + return `New recording captured for media ${resource.mediaId}`; + }, + async hydrate(resource) { + const results = await this.app.getInsights({ + params: { + uploadType: "recorder", + mediaId: resource.mediaId, + page: 0, + pageSize: 1, + }, + }); + return this.app.firstResult(results, resource); + }, + }, + sampleEmit, +}; diff --git a/components/speak_ai/sources/new-recording/test-event.mjs b/components/speak_ai/sources/new-recording/test-event.mjs new file mode 100644 index 0000000000000..b4e34ea183550 --- /dev/null +++ b/components/speak_ai/sources/new-recording/test-event.mjs @@ -0,0 +1,21 @@ +export default { + "name": "Recording for Recorder Tue Jul 02 2024 - 2024-10-24 15:59:57", + "description": "Client's recording for Recorder Tue Jul 02 2024", + "tags": [], + "duration": { + "inSecond": 17, + "end": "00:00:17.097", + "start": "00:00:01.760" + }, + "fields": [], + "mediaId": "f91abe4918c2", + "createdAt": "2024-10-24T19:59:57.164Z", + "originalCreatedAt": "2024-10-24T19:59:57.164Z", + "id": "f91abe4918c2", + "mediaUrl": "", + "recording": { + "recorderId": "970983c98875", + "recorderName": "Recorder Tue Jul 02 2024", + "questions": [] + } +}; diff --git a/components/speak_ai/sources/new-sentiment/new-sentiment.mjs b/components/speak_ai/sources/new-sentiment/new-sentiment.mjs new file mode 100644 index 0000000000000..3c9251e2258db --- /dev/null +++ b/components/speak_ai/sources/new-sentiment/new-sentiment.mjs @@ -0,0 +1,36 @@ +import common from "../common/base.mjs"; +import events from "../common/events.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "speak_ai-new-sentiment", + name: "New Sentiment Analysis (Instant)", + description: "Emit new event when Speak AI produces sentiment analysis for a media file (`media.analyzed`, `media.reanalyzed`). [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getEvents() { + return [ + events.MEDIA_ANALYZED, + events.MEDIA_REANALYZED, + ]; + }, + getSummary(resource) { + return `New sentiment analysis for media ${resource.mediaId}`; + }, + async hydrate(resource) { + const results = await this.app.getInsights({ + params: { + insightType: "sentiment", + mediaId: resource.mediaId, + pageSize: 1, + }, + }); + return this.app.firstResult(results, resource); + }, + }, + sampleEmit, +}; diff --git a/components/speak_ai/sources/new-sentiment/test-event.mjs b/components/speak_ai/sources/new-sentiment/test-event.mjs new file mode 100644 index 0000000000000..297e4784135f2 --- /dev/null +++ b/components/speak_ai/sources/new-sentiment/test-event.mjs @@ -0,0 +1,47 @@ +export default { + "name": "Multi - test. - 4", + "description": "", + "tags": [ + "meeting-assistant" + ], + "duration": { + "inSecond": 75.233333, + "end": "00:01:10.588", + "start": "00:00:02.960" + }, + "createdAt": "2024-02-09T16:59:15.280Z", + "mediaId": "01259ffdf07c", + "sentiment": [ + { + "document": { + "Compound": 23.75692307692308, + "Negative": 0, + "Neutral": 42.30769230769231, + "Positive": 57.692307692307686 + }, + "sentences": [ + { + "id": 1, + "instances": [ + { + "end": "00:00:09.705", + "endInSec": 9.705, + "start": "00:00:02.960", + "startInSec": 2.96 + } + ], + "score": { + "compound": 0.4019, + "neg": 0, + "neu": 0.816, + "pos": 0.184 + }, + "text": "Today, I'm working on speaker Library chart interface. " + } + ] + } + ], + "fields": [], + "originalCreatedAt": "2024-02-09T16:59:15.914Z", + "id": "01259ffdf07c" +}; diff --git a/components/speak_ai/sources/new-transcription/new-transcription.mjs b/components/speak_ai/sources/new-transcription/new-transcription.mjs new file mode 100644 index 0000000000000..4bd3faa3de793 --- /dev/null +++ b/components/speak_ai/sources/new-transcription/new-transcription.mjs @@ -0,0 +1,36 @@ +import common from "../common/base.mjs"; +import events from "../common/events.mjs"; +import sampleEmit from "./test-event.mjs"; + +export default { + ...common, + key: "speak_ai-new-transcription", + name: "New Automated Transcription (Instant)", + description: "Emit new event when Speak AI finishes transcribing and analyzing a media file (`media.analyzed`, `media.reanalyzed`). [See the documentation](https://docs.speakai.co/).", + version: "0.0.1", + type: "source", + dedupe: "unique", + methods: { + ...common.methods, + getEvents() { + return [ + events.MEDIA_ANALYZED, + events.MEDIA_REANALYZED, + ]; + }, + getSummary(resource) { + return `New transcription for media ${resource.mediaId}`; + }, + async hydrate(resource) { + const results = await this.app.getInsights({ + params: { + insightType: "transcript", + mediaId: resource.mediaId, + pageSize: 1, + }, + }); + return this.app.firstResult(results, resource); + }, + }, + sampleEmit, +}; diff --git a/components/speak_ai/sources/new-transcription/test-event.mjs b/components/speak_ai/sources/new-transcription/test-event.mjs new file mode 100644 index 0000000000000..8d71f6b073ec5 --- /dev/null +++ b/components/speak_ai/sources/new-transcription/test-event.mjs @@ -0,0 +1,53 @@ +export default { + "insight": { + "transcript": [ + { + "id": 1, + "text": "The best performance?", + "confidence": 0.9503, + "speakerId": 1, + "language": "en-US", + "instances": [ + { + "adjustedStart": "0:00:00.19", + "adjustedEnd": "0:00:02.56", + "start": "0:00:00.19", + "end": "0:00:02.56" + } + ] + }, + { + "id": 2, + "text": "Winner.", + "confidence": 0.7239, + "speakerId": 1, + "language": "en-US", + "instances": [ + { + "adjustedStart": "0:00:03.51", + "adjustedEnd": "0:00:04.24", + "start": "0:00:03.51", + "end": "0:00:04.24" + } + ] + } + ] + }, + "duration": { + "inSecond": 556, + "start": "0:00:00", + "end": "0:09:16.32299" + }, + "description": "8", + "tags": [ + "Pipedream", + "YouTube", + "Public Speech" + ], + "createdAt": "2022-12-12T20:03:48.857Z", + "name": "Christopher Judge Best Performance Award Speech", + "mediaId": "f5bae4321dff", + "originalCreatedAt": "2022-12-12T20:03:48.863Z", + "id": "f5bae4321dff", + "transcriptText": "The best performance? Winner. Christopher judge." +}; diff --git a/components/speak_ai/speak_ai.app.mjs b/components/speak_ai/speak_ai.app.mjs index 65b30467db45b..6f233b7fef98a 100644 --- a/components/speak_ai/speak_ai.app.mjs +++ b/components/speak_ai/speak_ai.app.mjs @@ -58,6 +58,17 @@ export default { })); }, }, + prompt: { + type: "string", + label: "Prompt", + description: "The instruction or question to run against your selected folder and/or media, e.g. `Summarize the key action items from this transcript`.", + }, + mediaIds: { + type: "string[]", + label: "Media IDs", + description: "One or more Speak AI media IDs to include as context for the prompt. Each is the media item's unique ID — get it from the **Find Media** action, the `mediaId` field on a media webhook event, or the media item in the Speak AI app.", + optional: true, + }, }, methods: { getUrl(path) { @@ -123,5 +134,96 @@ export default { ...args, }); }, + /** + * Subscribes a Speak AI webhook so events are delivered to the given callback URL. + * @param {object} [opts={}] - Options for the request. + * @param {object} opts.data - Subscription payload merged over the defaults + * (`source`, `description`); must include `callbackUrl` and `events`. + * @returns {Promise} The API response containing the created `webhookId`. + */ + subscribeWebhook({ + data, ...args + } = {}) { + return this.post({ + path: "/webhook", + data: { + source: "pipedream", + description: "Pipedream integration", + ...data, + }, + ...args, + }); + }, + /** + * Removes a previously created Speak AI webhook subscription. + * @param {object} [opts={}] - Options for the request. + * @param {string} opts.webhookId - The ID of the webhook to delete. + * @returns {Promise} The API response for the delete request. + */ + unsubscribeWebhook({ + webhookId, ...args + } = {}) { + return this.delete({ + path: `/webhook/${webhookId}`, + ...args, + }); + }, + /** + * Retrieves media insights (transcripts, sentiment, media items) from the apps endpoint. + * @param {object} [args={}] - Request options such as `params` and `$`. + * @returns {Promise} The insights response for the requested media. + */ + getInsights(args = {}) { + return this._makeRequest({ + path: "/apps/insights", + ...args, + }); + }, + /** + * Retrieves exported media assets (e.g. SRT/VTT captions) from the apps endpoint. + * @param {object} [args={}] - Request options such as `params`, `headers`, and `$`. + * @returns {Promise} The export response for the requested media. + */ + getExport(args = {}) { + return this._makeRequest({ + path: "/apps/export", + ...args, + }); + }, + /** + * Retrieves the AI Chat response history from the apps endpoint. + * @param {object} [args={}] - Request options such as `params`, `headers`, and `$`. + * @returns {Promise} The response containing the prompt history list. + */ + getPromptsHistory(args = {}) { + return this._makeRequest({ + path: "/apps/prompts/history", + ...args, + }); + }, + /** + * Runs a Speak AI Chat against a folder and/or specific media. + * @param {object} [args={}] - Request options; `data` carries the prompt payload. + * @returns {Promise} The API response for the submitted prompt. + */ + runPrompt(args = {}) { + return this.post({ + path: "/prompt", + ...args, + }); + }, + /** + * Normalizes an apps-endpoint response (insights/export) to a single resource. + * These endpoints may return either an array of results or a single object. + * @param {object|object[]} results - The raw response from an apps endpoint. + * @param {object} [fallback] - Value returned when no result is present. + * @returns {object} The first available result, or `fallback` when none exists. + */ + firstResult(results, fallback) { + const first = Array.isArray(results) + ? results[0] + : results; + return first || fallback; + }, }, };