diff --git a/package-lock.json b/package-lock.json index afc642c49ff..3bf5ae161d5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -350,7 +350,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" }, @@ -394,7 +393,6 @@ } ], "license": "MIT", - "peer": true, "engines": { "node": ">=18" } @@ -2305,7 +2303,6 @@ "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@types/body-parser": "*", "@types/express-serve-static-core": "^5.0.0", @@ -2501,7 +2498,6 @@ "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", "license": "MIT", - "peer": true, "dependencies": { "csstype": "^3.2.2" } @@ -2595,7 +2591,6 @@ "integrity": "sha512-BtE0k6cjwjLZoZixN0t5AKP0kSzlGu7FctRXYuPAm//aaiZhmfq1JwdYpYr1brzEspYyFeF+8XF5j2VK6oalrA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.54.0", "@typescript-eslint/types": "8.54.0", @@ -2887,7 +2882,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4491,7 +4485,6 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -5977,7 +5970,6 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.12.tgz", "integrity": "sha512-p1JfQMKaceuCbpJKAPKVqyqviZdS0eUxH9v82oWo1kb9xjQ5wA6iP3FNVAPDFlz5/p7d45lO+BpSk1tuSZMF4Q==", "license": "MIT", - "peer": true, "engines": { "node": ">=16.9.0" } @@ -8288,7 +8280,6 @@ "resolved": "https://registry.npmjs.org/react/-/react-19.2.4.tgz", "integrity": "sha512-9nfp2hYpCwOjAN+8TZFGhtWEwgvWHXqESH8qT89AT/lWklpLON22Lc8pEtnpsZz7VmawabSU0gCjnj8aC0euHQ==", "license": "MIT", - "peer": true, "engines": { "node": ">=0.10.0" } @@ -8867,8 +8858,7 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/signal-polyfill/-/signal-polyfill-0.2.2.tgz", "integrity": "sha512-p63Y4Er5/eMQ9RHg0M0Y64NlsQKpiu6MDdhBXpyywRuWiPywhJTpKJ1iB5K2hJEbFZ0BnDS7ZkJ+0AfTuL37Rg==", - "license": "Apache-2.0", - "peer": true + "license": "Apache-2.0" }, "node_modules/signal-utils": { "version": "0.21.1", @@ -9668,7 +9658,6 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -10263,7 +10252,6 @@ "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "devOptional": true, "license": "ISC", - "peer": true, "bin": { "yaml": "bin.mjs" }, @@ -10388,7 +10376,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.5.tgz", "integrity": "sha512-k7Nwx6vuWx1IJ9Bjuf4Zt1PEllcwe7cls3VNzm4CQ1/hgtFUK2bRNG3rvnpPUhFjmqJKAKtjV576KnUkHocg/g==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/packages/opal-backend/opal_backend/declarations/notebooklm.functions.json b/packages/opal-backend/opal_backend/declarations/notebooklm.functions.json index 3ef10b18f27..3f22a53421b 100644 --- a/packages/opal-backend/opal_backend/declarations/notebooklm.functions.json +++ b/packages/opal-backend/opal_backend/declarations/notebooklm.functions.json @@ -124,5 +124,80 @@ ], "additionalProperties": false } + }, + { + "name": "notebooklm_get_source", + "description": "Gets details and content of a specific source document within a NotebookLM notebook.", + "parametersJsonSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "notebook_id": { + "type": "string", + "description": "The NotebookLM notebook ID. Extract this from the NotebookLM URL by taking the ID after \"https://notebooklm.google.com/notebook/\". For example, if the URL is \"https://notebooklm.google.com/notebook/abc123\", pass \"abc123\" as the notebook_id." + }, + "source_id": { + "type": "string", + "description": "The ID of the source within the notebook." + }, + "task_id": { + "type": "string" + }, + "status_update": { + "type": "string", + "description": "A status update to show in the UI that provides more detail on the reason why this function was called.\n \n For example, \"Retrieving source document\", \"Reading file contents\", etc." + } + }, + "required": [ + "notebook_id", + "source_id", + "status_update" + ], + "additionalProperties": false + }, + "responseJsonSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "source": { + "type": "object", + "properties": { + "name": { "type": "string", "description": "The resource name of the source" }, + "display_name": { "type": "string", "description": "The display name of the source" }, + "state": { "type": "string", "description": "State of the source processing" }, + "original_mime_type": { "type": "string", "description": "The original mime type of the source document" }, + "user_drive_source_status": { "type": "string", "description": "The status of user drive source document" }, + "user_raw_source": { + "type": "object", + "description": "Details and contents of the raw source document.", + "properties": { + "blobstore_content": { + "type": "object", + "description": "GCS blob details for the source document.", + "properties": { + "blob_id": { "type": "string", "description": "The unique ID of the source document stored in GCS." } + }, + "required": [ + "blob_id" + ], + "additionalProperties": false + }, + "serving_url": { "type": "string", "description": "The temporary serving URL of the source document." }, + "download_url": { "type": "string", "description": "The download URL of the source document." } + }, + "additionalProperties": false + } + }, + "required": [ + "name" + ], + "additionalProperties": true + } + }, + "required": [ + "source" + ], + "additionalProperties": false + } } ] diff --git a/packages/opal-backend/opal_backend/declarations/notebooklm.instruction.md b/packages/opal-backend/opal_backend/declarations/notebooklm.instruction.md index fb0102787fe..e49998ebcbd 100644 --- a/packages/opal-backend/opal_backend/declarations/notebooklm.instruction.md +++ b/packages/opal-backend/opal_backend/declarations/notebooklm.instruction.md @@ -4,16 +4,21 @@ You have access to NotebookLM notebooks as knowledge sources. When the objective references a NotebookLM notebook (indicated by a URL like https://notebooklm.google.com/notebook/{notebook_id}), you can: -1. Use "notebooklm_generate_answer" to generate a comprehensive answer - to a question using the notebook's AI chat functionality. This is useful when - you need the notebook to synthesize information and provide a direct answer. +1. Use "notebooklm_generate_answer" to generate a comprehensive answer to a + question using the notebook's AI chat functionality. This is useful when you + need the notebook to synthesize information and provide a direct answer. 2. Use "notebooklm_retrieve_relevant_chunks" to retrieve relevant source material from the notebook (text, images, or audio) based on a query. This is useful when you want to retrieve source documents/content, not just get a - summary (use this like a RAG system for the notebook content). Each - retrieval is limited to a token budget, so it may be necessary to make multiple - more narrow queries if you need more information. + summary (use this like a RAG system for the notebook content). Each retrieval + is limited to a token budget, so it may be necessary to make multiple more + narrow queries if you need more information. -The URL format is "https://notebooklm.google.com/notebook/{notebook_id}" where +3. Use "notebooklm_get_source" to retrieve to retrieve complete source material + from the notebook (text, images, or audio) that was referenced in the query. + This is useful when you want to retrieve the complete source + documents/content, not just get a small chunk of the source. + +The URL format is "https://notebooklm.google.com/notebook/{notebook_id}" where "{notebook_id}" is the ID you should pass to the function. diff --git a/packages/opal-backend/opal_backend/declarations/notebooklm.metadata.json b/packages/opal-backend/opal_backend/declarations/notebooklm.metadata.json index 75112e4f8a3..9f7035a5a92 100644 --- a/packages/opal-backend/opal_backend/declarations/notebooklm.metadata.json +++ b/packages/opal-backend/opal_backend/declarations/notebooklm.metadata.json @@ -4,5 +4,8 @@ }, { "name": "notebooklm_generate_answer" + }, + { + "name": "notebooklm_get_source" } ] diff --git a/packages/visual-editor/src/a2/agent/functions/generated/notebooklm.ts b/packages/visual-editor/src/a2/agent/functions/generated/notebooklm.ts index 10c4045f42c..15db0a25490 100644 --- a/packages/visual-editor/src/a2/agent/functions/generated/notebooklm.ts +++ b/packages/visual-editor/src/a2/agent/functions/generated/notebooklm.ts @@ -39,6 +39,30 @@ export type NotebooklmGenerateAnswerResponse = { answer: string; }; +export type NotebooklmGetSourceParams = { + notebook_id: string; + source_id: string; + task_id?: string; + status_update: string; +}; + +export type NotebooklmGetSourceResponse = { + source: { + name: string; + display_name?: string; + state?: string; + original_mime_type?: string; + user_drive_source_status?: string; + user_raw_source?: { + blobstore_content?: { + blob_id: string; + }; + serving_url?: string; + download_url?: string; + }; + }; +}; + export const declarations: FunctionDeclaration[] = [ { "name": "notebooklm_retrieve_relevant_chunks", @@ -165,12 +189,112 @@ export const declarations: FunctionDeclaration[] = [ ], "additionalProperties": false } + }, + { + "name": "notebooklm_get_source", + "description": "Gets details and content of a specific source document within a NotebookLM notebook.", + "parametersJsonSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "notebook_id": { + "type": "string", + "description": "The NotebookLM notebook ID. Extract this from the NotebookLM URL by taking the ID after \"https://notebooklm.google.com/notebook/\". For example, if the URL is \"https://notebooklm.google.com/notebook/abc123\", pass \"abc123\" as the notebook_id." + }, + "source_id": { + "type": "string", + "description": "The ID of the source within the notebook." + }, + "task_id": { + "type": "string" + }, + "status_update": { + "type": "string", + "description": "A status update to show in the UI that provides more detail on the reason why this function was called.\n \n For example, \"Retrieving source document\", \"Reading file contents\", etc." + } + }, + "required": [ + "notebook_id", + "source_id", + "status_update" + ], + "additionalProperties": false + }, + "responseJsonSchema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "source": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "The resource name of the source" + }, + "display_name": { + "type": "string", + "description": "The display name of the source" + }, + "state": { + "type": "string", + "description": "State of the source processing" + }, + "original_mime_type": { + "type": "string", + "description": "The original mime type of the source document" + }, + "user_drive_source_status": { + "type": "string", + "description": "The status of user drive source document" + }, + "user_raw_source": { + "type": "object", + "description": "Details and contents of the raw source document.", + "properties": { + "blobstore_content": { + "type": "object", + "description": "GCS blob details for the source document.", + "properties": { + "blob_id": { + "type": "string", + "description": "The unique ID of the source document stored in GCS." + } + }, + "required": [ + "blob_id" + ], + "additionalProperties": false + }, + "serving_url": { + "type": "string", + "description": "The temporary serving URL of the source document." + }, + "download_url": { + "type": "string", + "description": "The download URL of the source document." + } + }, + "additionalProperties": false + } + }, + "required": [ + "name" + ], + "additionalProperties": true + } + }, + "required": [ + "source" + ], + "additionalProperties": false + } } ]; export const metadata: Record = { "notebooklm_retrieve_relevant_chunks": {}, - "notebooklm_generate_answer": {} + "notebooklm_generate_answer": {}, + "notebooklm_get_source": {} }; -export const instruction: string = "## Using NotebookLM\n\nYou have access to NotebookLM notebooks as knowledge sources. When the objective\nreferences a NotebookLM notebook (indicated by a URL like\nhttps://notebooklm.google.com/notebook/{notebook_id}), you can:\n\n1. Use \"notebooklm_generate_answer\" to generate a comprehensive answer \n to a question using the notebook's AI chat functionality. This is useful when \n you need the notebook to synthesize information and provide a direct answer.\n\n2. Use \"notebooklm_retrieve_relevant_chunks\" to retrieve relevant source\n material from the notebook (text, images, or audio) based on a query. This is\n useful when you want to retrieve source documents/content, not just get a\n summary (use this like a RAG system for the notebook content). Each\n retrieval is limited to a token budget, so it may be necessary to make multiple\n more narrow queries if you need more information.\n\nThe URL format is \"https://notebooklm.google.com/notebook/{notebook_id}\" where \n\"{notebook_id}\" is the ID you should pass to the function."; +export const instruction: string = "## Using NotebookLM\n\nYou have access to NotebookLM notebooks as knowledge sources. When the objective\nreferences a NotebookLM notebook (indicated by a URL like\nhttps://notebooklm.google.com/notebook/{notebook_id}), you can:\n\n1. Use \"notebooklm_generate_answer\" to generate a comprehensive answer to a\n question using the notebook's AI chat functionality. This is useful when you\n need the notebook to synthesize information and provide a direct answer.\n\n2. Use \"notebooklm_retrieve_relevant_chunks\" to retrieve relevant source\n material from the notebook (text, images, or audio) based on a query. This is\n useful when you want to retrieve source documents/content, not just get a\n summary (use this like a RAG system for the notebook content). Each retrieval\n is limited to a token budget, so it may be necessary to make multiple more\n narrow queries if you need more information.\n\n3. Use \"notebooklm_get_source\" to retrieve to retrieve complete source material\n from the notebook (text, images, or audio) that was referenced in the query.\n This is useful when you want to retrieve the complete source\n documents/content, not just get a small chunk of the source.\n\nThe URL format is \"https://notebooklm.google.com/notebook/{notebook_id}\" where\n\"{notebook_id}\" is the ID you should pass to the function."; diff --git a/packages/visual-editor/src/a2/agent/functions/notebooklm.ts b/packages/visual-editor/src/a2/agent/functions/notebooklm.ts index f991dab2213..5e680460768 100644 --- a/packages/visual-editor/src/a2/agent/functions/notebooklm.ts +++ b/packages/visual-editor/src/a2/agent/functions/notebooklm.ts @@ -26,6 +26,7 @@ import { instruction, type NotebooklmRetrieveRelevantChunksParams, type NotebooklmGenerateAnswerParams, + type NotebooklmGetSourceParams, } from "./generated/notebooklm.js"; export { getNotebookLMFunctionGroup }; @@ -121,8 +122,10 @@ function getNotebookLMFunctionGroup( const chunks = response.sourceContexts.flatMap( (sourceContext: SourceContext) => sourceContext.chunks.map((chunk) => { - const { textContent, mediaPaths, errors } = - processContentPieces(chunk.content?.pieces ?? [], fileSystem); + const { textContent, mediaPaths, errors } = processContentPieces( + chunk.content?.pieces ?? [], + fileSystem + ); allErrors.push(...errors); return { source_name: sourceContext.sourceName, @@ -169,5 +172,42 @@ function getNotebookLMFunctionGroup( ); } }, + + notebooklm_get_source: async ({ + notebook_id, + source_id, + task_id, + status_update, + }: NotebooklmGetSourceParams) => { + taskTreeManager.setInProgress(task_id, status_update); + + try { + const source = await notebookLmApiClient.getSource({ + name: `notebooks/${notebook_id}/sources/${source_id}`, + }); + + return { + source: { + name: source.name, + display_name: source.displayName, + state: source.state, + user_raw_source: source.userRawSource + ? { + blobstore_content: source.userRawSource?.storedData?.handle + ? { blob_id: source.userRawSource.storedData.handle.split("/").at(-1)! } + : undefined, + serving_url: source.userRawSource?.servingUrl, + download_url: source.userRawSource?.downloadUrl, + } + : undefined, + original_mime_type: source.originalMimeType, + }, + }; + } catch (error) { + return err( + `Failed to get source from notebook: ${(error as Error).message}` + ); + } + }, }); } diff --git a/packages/visual-editor/src/sca/services/notebooklm-api-client.ts b/packages/visual-editor/src/sca/services/notebooklm-api-client.ts index 5ac1f03a730..8ecf78986b8 100644 --- a/packages/visual-editor/src/sca/services/notebooklm-api-client.ts +++ b/packages/visual-editor/src/sca/services/notebooklm-api-client.ts @@ -101,15 +101,30 @@ export interface TextContent { contentType: TextContentType; } +// eslint-disable-next-line local-rules/no-exported-types-outside-types-ts +export interface StoredData { + handle?: string; + mimeType?: string; +} + +// eslint-disable-next-line local-rules/no-exported-types-outside-types-ts +export interface UserRawSource { + storedData?: StoredData; + servingUrl?: string; + downloadUrl?: string; +} + // eslint-disable-next-line local-rules/no-exported-types-outside-types-ts export interface Source { name: string; displayName?: string; createTime?: string; state?: SourceState; + userRawSource?: UserRawSource; webContent?: WebContent; blobstoreContent?: BlobstoreContent; textContent?: TextContent; + originalMimeType?: string; } // ============================================================================= @@ -255,6 +270,13 @@ export interface GetNotebookRequest { notebookExpansion?: NotebookExpansion; } +/** Request for GetSource RPC. */ +// eslint-disable-next-line local-rules/no-exported-types-outside-types-ts +export interface GetSourceRequest { + name: string; + provenance?: Provenance; +} + /** Request for RetrieveRelevantChunks RPC. */ // eslint-disable-next-line local-rules/no-exported-types-outside-types-ts export interface RetrieveRelevantChunksRequest { @@ -425,6 +447,45 @@ export class NotebookLmApiClient { return (await response.json()) as Notebook; } + /** + * Gets details of a specific Source resource. + * @param request - The request containing the source name (e.g. "notebooks/{notebook_id}/sources/{source_id}") + */ + async getSource(request: GetSourceRequest): Promise { + // name format: "notebooks/{notebook_id}/sources/{source_id}" + const url = this.#backendApiBaseUrl + ? new URL(`v1beta1/${request.name}`, this.#backendApiBaseUrl) + : new URL(`v1/${request.name}`, this.#apiBaseUrl); + + let response: Response; + + if (this.#backendApiBaseUrl) { + response = await this.#fetchWithCreds(url, { + method: "GET", + headers: { + "content-type": "application/json", + }, + }); + } else { + // Add provenance as query params with dot notation + this.#appendProvenanceParams(url, request.provenance); + + response = await this.#fetchWithCreds(url, { + method: "GET", + headers: { + "content-type": "application/json", + }, + }); + } + + if (!response.ok) { + throw new Error(`Failed to get source: ${response.statusText}`); + } + + const json = await response.json(); + return json as Source; + } + /** * Returns relevant chunks of content from a notebook's sources. * Performs a relevance-based search against the notebook's content. diff --git a/packages/visual-editor/tests/sca/helpers/fake-notebooklm-api.ts b/packages/visual-editor/tests/sca/helpers/fake-notebooklm-api.ts index 83e90ae6ce4..4fe26171493 100644 --- a/packages/visual-editor/tests/sca/helpers/fake-notebooklm-api.ts +++ b/packages/visual-editor/tests/sca/helpers/fake-notebooklm-api.ts @@ -10,6 +10,8 @@ import type { ListNotebooksRequest, ListNotebooksResponse, GetNotebookRequest, + GetSourceRequest, + Source, RetrieveRelevantChunksRequest, RetrieveRelevantChunksResponse, ListNotebookPermissionsRequest, @@ -119,6 +121,21 @@ export class FakeNotebookLmApiClient implements PublicInterface { + this.#recordAndMaybeThrow("getSource", [request]); + // Search inside the in-memory notebooks for the source + for (const notebook of this.notebooks.values()) { + if (notebook.sources) { + for (const src of notebook.sources) { + if (src.name === request.name) { + return src; + } + } + } + } + throw new Error(`Source not found: ${request.name}`); + } + async retrieveRelevantChunks( request: RetrieveRelevantChunksRequest ): Promise {