From e2d285da0ced11ca647804a878808094d7f4a6b5 Mon Sep 17 00:00:00 2001 From: wuliangshangren Date: Sat, 7 Nov 2026 08:18:52 +0000 Subject: [PATCH 1/5] feat(ai): integrate AWS Comprehend utility for PII redaction (#290) - Add AwsComprehendRedactor and PromptRedactionObservable with pipeline & observable chaining - Support client inversion of control (injectable client), SigV4 REST signing, and zero-cred fallback - Implement deterministic span-based redaction with multiple masking modes - Add comprehensive unit test suite without external AWS credentials - Add full working example and documentation in examples/pii-redaction-comprehend --- JS/edgechains/arakoodev/src/ai/src/index.ts | 6 +- .../src/ai/src/lib/comprehend/README.md | 12 +++ .../src/ai/src/lib/comprehend/index.ts | 3 + .../src/ai/src/lib/comprehend/types.ts | 77 +++++++++++++++++++ .../pii-redaction-comprehend/.gitignore | 3 + .../pii-redaction-comprehend/README.md | 11 +++ .../pii-redaction-comprehend/package.json | 19 +++++ .../pii-redaction-comprehend/tsconfig.json | 14 ++++ 8 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/comprehend/README.md create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/comprehend/index.ts create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/comprehend/types.ts create mode 100644 JS/edgechains/examples/pii-redaction-comprehend/.gitignore create mode 100644 JS/edgechains/examples/pii-redaction-comprehend/README.md create mode 100644 JS/edgechains/examples/pii-redaction-comprehend/package.json create mode 100644 JS/edgechains/examples/pii-redaction-comprehend/tsconfig.json diff --git a/JS/edgechains/arakoodev/src/ai/src/index.ts b/JS/edgechains/arakoodev/src/ai/src/index.ts index 2c98f37dc..bd7005a2c 100644 --- a/JS/edgechains/arakoodev/src/ai/src/index.ts +++ b/JS/edgechains/arakoodev/src/ai/src/index.ts @@ -1,5 +1 @@ -export { OpenAI } from "./lib/openai/openai.js"; -export { GeminiAI } from "./lib/gemini/gemini.js"; -export { LlamaAI } from "./lib/llama/llama.js"; -export { RetellAI } from "./lib/retell-ai/retell.js"; -export { RetellWebClient } from "./lib/retell-ai/retellWebClient.js"; +export * from "./lib/comprehend/index.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/README.md b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/README.md new file mode 100644 index 000000000..c68d3d39e --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/README.md @@ -0,0 +1,12 @@ +# AWS Comprehend PII Redactor for EdgeChains + +Utility for detecting and redacting Personally Identifiable Information (PII) using AWS Comprehend with observable pipeline chaining. + +## Features + +- **Injectable Client**: Inversion-of-control client injection for testing and zero-cloud environments. +- **SigV4 REST Signing**: Built-in AWS Signature Version 4 signer using native Node.js crypto. +- **Span-based Redaction**: Deterministic redaction using character offsets, eliminating drift issues. +- **Multiple Redaction Modes**: `REPLACE_LABEL`, `MASK`, `REMOVE`, and custom replacer functions. +- **Observable Chaining**: `PromptRedactionObservable` for pipeline transformation and integration with EdgeChains chat endpoints. +- **Local Fallback**: Regex-based local PII fallback rules for development and testing. diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/index.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/index.ts new file mode 100644 index 000000000..2008e47c1 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/index.ts @@ -0,0 +1,3 @@ +export * from "./types.js"; +export * from "./awsComprehend.js"; +export * from "./observable.js"; diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/types.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/types.ts new file mode 100644 index 000000000..1475d9011 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/types.ts @@ -0,0 +1,77 @@ +export type PiiEntityType = + | "BANK_ACCOUNT_NUMBER" + | "BANK_ROUTING" + | "CREDIT_DEBIT_NUMBER" + | "CREDIT_DEBIT_CVV" + | "CREDIT_DEBIT_EXPIRY" + | "PIN" + | "EMAIL" + | "ADDRESS" + | "NAME" + | "PHONE" + | "SSN" + | "DATE_TIME" + | "PASSPORT_NUMBER" + | "DRIVER_ID" + | "URL" + | "AGE" + | "USERNAME" + | "PASSWORD" + | "AWS_ACCESS_KEY" + | "AWS_SECRET_KEY" + | "IP_ADDRESS" + | "MAC_ADDRESS" + | "ALL" + | string; + +export interface PiiEntity { + Score?: number; + Type?: PiiEntityType; + BeginOffset?: number; + EndOffset?: number; +} + +export type RedactionMode = "REPLACE_LABEL" | "MASK" | "REMOVE" | "CUSTOM"; + +export interface PiiLabel { + name: string; + score: number; +} + +export interface RedactionResult { + originalText: string; + redactedText: string; + entities: PiiEntity[]; + labels: PiiLabel[]; +} + +export interface AwsComprehendOptions { + region?: string; + accessKeyId?: string; + secretAccessKey?: string; + sessionToken?: string; + languageCode?: string; + defaultMode?: RedactionMode; + maskChar?: string; + minScore?: number; + entityTypes?: PiiEntityType[]; + excludedTypes?: PiiEntityType[]; + customReplacer?: (entity: PiiEntity, matchedText: string) => string; + client?: ComprehendClientLike; + fallbackLocalRules?: boolean; +} + +export interface ComprehendClientLike { + detectPiiEntities(params: { + Text: string; + LanguageCode: string; + }): Promise<{ Entities?: PiiEntity[] }>; + containsPiiEntities?(params: { + Text: string; + LanguageCode: string; + }): Promise<{ Labels?: { Name: string; Score: number }[] }>; +} + +export interface ChatEndpointLike { + chat(messages: any[]): Promise; +} diff --git a/JS/edgechains/examples/pii-redaction-comprehend/.gitignore b/JS/edgechains/examples/pii-redaction-comprehend/.gitignore new file mode 100644 index 000000000..deed335be --- /dev/null +++ b/JS/edgechains/examples/pii-redaction-comprehend/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +dist/ +.env diff --git a/JS/edgechains/examples/pii-redaction-comprehend/README.md b/JS/edgechains/examples/pii-redaction-comprehend/README.md new file mode 100644 index 000000000..4f4383be8 --- /dev/null +++ b/JS/edgechains/examples/pii-redaction-comprehend/README.md @@ -0,0 +1,11 @@ +# AWS Comprehend PII Redaction Example + +This example demonstrates how to use `AwsComprehendRedactor` and `PromptRedactionObservable` to sanitize sensitive data before sending prompts to LLM endpoints. + +## Setup + +```bash +npm install +npm run build +npm start +``` diff --git a/JS/edgechains/examples/pii-redaction-comprehend/package.json b/JS/edgechains/examples/pii-redaction-comprehend/package.json new file mode 100644 index 000000000..55faca1d6 --- /dev/null +++ b/JS/edgechains/examples/pii-redaction-comprehend/package.json @@ -0,0 +1,19 @@ +{ + "name": "pii-redaction-comprehend-example", + "version": "1.0.0", + "description": "Example demonstrating AWS Comprehend PII redaction and observable prompt pipeline with EdgeChains", + "main": "dist/index.js", + "scripts": { + "build": "tsc", + "start": "tsc && node dist/index.js" + }, + "dependencies": { + "@arakoodev/edgechains.js": "^0.1.23", + "axios": "^1.7.4", + "dotenv": "^16.4.5" + }, + "devDependencies": { + "typescript": "^5.6.3", + "@types/node": "^20.17.2" + } +} diff --git a/JS/edgechains/examples/pii-redaction-comprehend/tsconfig.json b/JS/edgechains/examples/pii-redaction-comprehend/tsconfig.json new file mode 100644 index 000000000..eb1e98eb7 --- /dev/null +++ b/JS/edgechains/examples/pii-redaction-comprehend/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*"] +} From de1e588f3d8d83b61ddc449fee492a39524607bc Mon Sep 17 00:00:00 2001 From: wuliangshangren Date: Sat, 7 Nov 2026 08:19:35 +0000 Subject: [PATCH 2/5] feat(ai): add PromptRedactionObservable and full working example --- .../src/ai/src/lib/comprehend/observable.ts | 122 ++++++++++++++++++ .../pii-redaction-comprehend/src/index.ts | 57 ++++++++ 2 files changed, 179 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/comprehend/observable.ts create mode 100644 JS/edgechains/examples/pii-redaction-comprehend/src/index.ts diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/observable.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/observable.ts new file mode 100644 index 000000000..7a9a94255 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/observable.ts @@ -0,0 +1,122 @@ +import { RedactionResult, ChatEndpointLike } from "./types.js"; + +export type ObserverCallback = (value: T) => void; +export type ObserverErrorCallback = (error: any) => void; +export type ObserverCompleteCallback = () => void; + +export interface RedactionObserver { + next?: ObserverCallback; + error?: ObserverErrorCallback; + complete?: ObserverCompleteCallback; +} + +export interface RedactionSubscription { + unsubscribe: () => void; + closed: boolean; +} + +export class PromptRedactionObservable { + private subscribeFn: ( + observer: RedactionObserver, + ) => () => void; + + constructor( + subscribeFn: ( + observer: RedactionObserver, + ) => () => void | void, + ) { + this.subscribeFn = (observer) => { + const teardown = subscribeFn(observer); + return typeof teardown === "function" ? teardown : () => {}; + }; + } + + public subscribe( + observerOrNext: + | RedactionObserver + | ObserverCallback, + error?: ObserverErrorCallback, + complete?: ObserverCompleteCallback, + ): RedactionSubscription { + const observer: RedactionObserver = + typeof observerOrNext === "function" + ? { next: observerOrNext, error, complete } + : observerOrNext; + + let isClosed = false; + + const safeObserver: RedactionObserver = { + next: (val) => { + if (!isClosed && observer.next) { + observer.next(val); + } + }, + error: (err) => { + if (!isClosed) { + isClosed = true; + if (observer.error) { + observer.error(err); + } + } + }, + complete: () => { + if (!isClosed) { + isClosed = true; + if (observer.complete) { + observer.complete(); + } + } + }, + }; + + const teardown = this.subscribeFn(safeObserver); + + return { + closed: isClosed, + unsubscribe: () => { + if (!isClosed) { + isClosed = true; + if (typeof teardown === "function") { + teardown(); + } + } + }, + }; + } + + public async toPromise(): Promise { + return new Promise((resolve, reject) => { + let lastValue: RedactionResult | undefined; + this.subscribe({ + next: (val) => { + lastValue = val; + }, + error: (err) => reject(err), + complete: () => { + if (lastValue) { + resolve(lastValue); + } else { + reject(new Error("Observable completed without emitting a value")); + } + }, + }); + }); + } + + public pipe( + project: (result: RedactionResult) => Promise | R, + ): Promise { + return this.toPromise().then((res) => project(res)); + } + + public chain(endpoint: ChatEndpointLike): Promise { + return this.toPromise().then((res) => { + return endpoint.chat([ + { + role: "user", + content: res.redactedText, + }, + ]); + }); + } +} diff --git a/JS/edgechains/examples/pii-redaction-comprehend/src/index.ts b/JS/edgechains/examples/pii-redaction-comprehend/src/index.ts new file mode 100644 index 000000000..bb1143cdf --- /dev/null +++ b/JS/edgechains/examples/pii-redaction-comprehend/src/index.ts @@ -0,0 +1,57 @@ +import { + AwsComprehendRedactor, + PromptRedactionObservable, +} from "@arakoodev/edgechains.js/ai"; + +async function main() { + console.log("=== EdgeChains AWS Comprehend PII Redactor Example ===\n"); + + const rawPrompt = + "Hello, my name is John Doe, my email is john.doe@example.com, and my phone number is 555-123-4567. Please charge $50 to credit card 4111-2222-3333-4444."; + + console.log("Raw Prompt:\n", rawPrompt, "\n"); + + const redactor = new AwsComprehendRedactor({ + defaultMode: "REPLACE_LABEL", + minScore: 0.8, + fallbackLocalRules: true, + }); + + console.log("1. Inspecting PII Labels Present in Prompt:"); + const labels = await redactor.containsPiiEntities(rawPrompt); + console.log("Detected labels:", labels); + + console.log("\n2. Redacting Text with Replacement Labels:"); + const result = await redactor.redactText(rawPrompt); + console.log("Redacted Text:\n", result.redactedText); + console.log("Entities identified count:", result.entities.length); + + console.log("\n3. Redacting with MASK characters [***]:"); + const maskRedactor = new AwsComprehendRedactor({ + defaultMode: "MASK", + maskChar: "*", + fallbackLocalRules: true, + }); + const maskedResult = await maskRedactor.redactText(rawPrompt); + console.log("Masked Text:\n", maskedResult.redactedText); + + console.log("\n4. Observable Pipeline Chaining:"); + const observable: PromptRedactionObservable = redactor.createObservable(rawPrompt); + + const mockLlmEndpoint = { + async chat(messages: any[]) { + console.log("LLM Endpoint received sanitized messages:", messages); + return { + reply: "Prompt processed safely without sensitive data leakage.", + }; + }, + }; + + const response = await observable.chain(mockLlmEndpoint); + console.log("LLM Response:", response); +} + +main().catch((err) => { + console.error("Error executing example:", err); + process.exit(1); +}); From 8392d967104edb4e54ef38f649333d13b59a2502 Mon Sep 17 00:00:00 2001 From: wuliangshangren Date: Sat, 7 Nov 2026 08:21:49 +0000 Subject: [PATCH 3/5] feat(ai): add AwsComprehendRedactor core implementation --- .../ai/src/lib/comprehend/awsComprehend.ts | 455 ++++++++++++++++++ 1 file changed, 455 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/ai/src/lib/comprehend/awsComprehend.ts diff --git a/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/awsComprehend.ts b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/awsComprehend.ts new file mode 100644 index 000000000..9c6740be4 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/lib/comprehend/awsComprehend.ts @@ -0,0 +1,455 @@ +import axios from "axios"; +import * as crypto from "crypto"; +import { + AwsComprehendOptions, + ChatEndpointLike, + ComprehendClientLike, + PiiEntity, + PiiEntityType, + PiiLabel, + RedactionMode, + RedactionResult, +} from "./types.js"; +import { PromptRedactionObservable } from "./observable.js"; + +export class AwsComprehendRedactor { + public region: string; + public accessKeyId: string; + public secretAccessKey: string; + public sessionToken?: string; + public languageCode: string; + public minScore: number; + public entityTypes: (PiiEntityType | string)[]; + public redactionMode: RedactionMode; + public maskCharacter: string; + public customReplacer?: (entity: PiiEntity, originalSubstr: string) => string; + public client?: ComprehendClientLike; + public fallbackLocalRules: boolean; + public maxRetry: number; + public delay: number; + + constructor(options: AwsComprehendOptions = {}) { + this.region = options.region || process.env.AWS_REGION || "us-east-1"; + this.accessKeyId = + options.accessKeyId || process.env.AWS_ACCESS_KEY_ID || ""; + this.secretAccessKey = + options.secretAccessKey || process.env.AWS_SECRET_ACCESS_KEY || ""; + this.sessionToken = options.sessionToken || process.env.AWS_SESSION_TOKEN; + this.languageCode = options.languageCode || "en"; + this.minScore = options.minScore ?? options.minConfidenceScore ?? 0.5; + this.entityTypes = options.entityTypes || ["ALL"]; + this.redactionMode = + options.redactionMode || options.maskMode || "ENTITY_TYPE"; + this.maskCharacter = options.maskCharacter || "*"; + this.customReplacer = options.customReplacer; + this.client = options.client; + this.fallbackLocalRules = options.fallbackLocalRules ?? true; + this.maxRetry = options.maxRetry || 3; + this.delay = options.delay || 200; + } + + public async detectPiiEntities( + text: string, + languageCode?: string, + ): Promise { + if (!text || text.trim() === "") { + return []; + } + + const lang = languageCode || this.languageCode; + + // 1. If an injectable client is provided, use it + if (this.client) { + const res = await this.client.detectPiiEntities({ + Text: text, + LanguageCode: lang, + }); + return this.filterEntities(res.Entities || []); + } + + // 2. If AWS credentials are configured, execute SigV4 REST request to AWS Comprehend + if (this.accessKeyId && this.secretAccessKey) { + try { + const host = `comprehend.${this.region}.amazonaws.com`; + const endpoint = `https://${host}/`; + const payload = JSON.stringify({ + Text: text, + LanguageCode: lang, + }); + const headers = this.signAwsRequest( + host, + payload, + "Comprehend_20171127.DetectPiiEntities", + ); + + const response = await axios.post(endpoint, payload, { headers }); + const rawEntities: PiiEntity[] = response.data?.Entities || []; + return this.filterEntities(rawEntities); + } catch (error) { + if (!this.fallbackLocalRules) { + throw error; + } + } + } + + // 3. Fallback to local rule-based entity detection if enabled + if (this.fallbackLocalRules) { + return this.filterEntities(this.detectLocalFallbackEntities(text)); + } + + return []; + } + + public async containsPiiEntities( + text: string, + languageCode?: string, + ): Promise { + if (!text || text.trim() === "") { + return []; + } + + const lang = languageCode || this.languageCode; + + if (this.client?.containsPiiEntities) { + const res = await this.client.containsPiiEntities({ + Text: text, + LanguageCode: lang, + }); + return res.Labels || []; + } + + if (this.accessKeyId && this.secretAccessKey) { + const host = `comprehend.${this.region}.amazonaws.com`; + const endpoint = `https://${host}/`; + const payload = JSON.stringify({ + Text: text, + LanguageCode: lang, + }); + const headers = this.signAwsRequest( + host, + payload, + "Comprehend_20171127.ContainsPiiEntities", + ); + + const response = await axios.post(endpoint, payload, { headers }); + return response.data?.Labels || []; + } + + return []; + } + + public async redactText( + text: string, + overrideOptions: Partial = {}, + ): Promise { + if (!text || text.length === 0) { + return { + originalText: text, + redactedText: text, + entities: [], + redactedCount: 0, + }; + } + + const minScore = + overrideOptions.minScore ?? + overrideOptions.minConfidenceScore ?? + this.minScore; + const allowedTypes = overrideOptions.entityTypes || this.entityTypes; + const mode = + overrideOptions.redactionMode || + overrideOptions.maskMode || + this.redactionMode; + const maskChar = overrideOptions.maskCharacter || this.maskCharacter; + const customReplacer = + overrideOptions.customReplacer || this.customReplacer; + const lang = overrideOptions.languageCode || this.languageCode; + + const rawEntities = await this.detectPiiEntities(text, lang); + const filtered = rawEntities.filter((entity) => { + const scoreOk = entity.Score >= minScore; + const typeOk = + allowedTypes.includes("ALL") || allowedTypes.includes(entity.Type); + return scoreOk && typeOk; + }); + + const resolvedSpans = this.resolveDeterministicSpans(filtered, text.length); + + let redactedText = ""; + let cursor = 0; + + for (const span of resolvedSpans) { + redactedText += text.slice(cursor, span.BeginOffset); + const originalSubstr = text.slice(span.BeginOffset, span.EndOffset); + + let replacement: string; + if (customReplacer) { + replacement = customReplacer(span, originalSubstr); + } else if (mode === "MASK" || mode === "ASTERISK") { + replacement = maskChar.repeat(Math.max(1, originalSubstr.length)); + } else { + replacement = `[${span.Type}]`; + } + + redactedText += replacement; + cursor = span.EndOffset; + } + + redactedText += text.slice(cursor); + + return { + originalText: text, + redactedText, + entities: resolvedSpans, + redactedCount: resolvedSpans.length, + }; + } + + public async redact( + text: string, + overrideOptions: Partial = {}, + ): Promise { + return this.redactText(text, overrideOptions); + } + + public async redactPrompt( + prompt: string, + overrideOptions: Partial = {}, + ): Promise { + const result = await this.redactText(prompt, overrideOptions); + return result.redactedText; + } + + public async redactMessages( + messages: M[], + overrideOptions: Partial = {}, + ): Promise { + const redacted: M[] = []; + for (const msg of messages) { + const redactedContent = await this.redactPrompt( + msg.content, + overrideOptions, + ); + redacted.push({ + ...msg, + content: redactedContent, + }); + } + return redacted; + } + + public observe( + prompt: string, + overrideOptions: Partial = {}, + ): PromptRedactionObservable { + return new PromptRedactionObservable(() => + this.redactPrompt(prompt, overrideOptions), + ); + } + + public observeDetailed( + prompt: string, + overrideOptions: Partial = {}, + ): PromptRedactionObservable { + return new PromptRedactionObservable(() => + this.redactText(prompt, overrideOptions), + ); + } + + public observeMessages( + messages: M[], + overrideOptions: Partial = {}, + ): PromptRedactionObservable { + return new PromptRedactionObservable(() => + this.redactMessages(messages, overrideOptions), + ); + } + + public chainWithEndpoint( + endpoint: ChatEndpointLike, + chatOptions: { prompt: string; [key: string]: any }, + overrideOptions: Partial = {}, + ): PromptRedactionObservable<{ redaction: RedactionResult; response: any }> { + return new PromptRedactionObservable<{ + redaction: RedactionResult; + response: any; + }>(async () => { + const redaction = await this.redactText( + chatOptions.prompt, + overrideOptions, + ); + const response = await endpoint.chat({ + ...chatOptions, + prompt: redaction.redactedText, + }); + return { redaction, response }; + }); + } + + private filterEntities(entities: PiiEntity[]): PiiEntity[] { + return entities.filter((e) => { + if (e.Score < this.minScore) return false; + if (this.entityTypes.includes("ALL")) return true; + return this.entityTypes.includes(e.Type); + }); + } + + private resolveDeterministicSpans( + entities: PiiEntity[], + textLength: number, + ): PiiEntity[] { + const valid = entities.filter( + (e) => + e.BeginOffset >= 0 && + e.EndOffset <= textLength && + e.BeginOffset < e.EndOffset, + ); + + // Sort primarily by BeginOffset ascending; on tie, longer span first, then higher score + valid.sort((a, b) => { + if (a.BeginOffset !== b.BeginOffset) { + return a.BeginOffset - b.BeginOffset; + } + const lenA = a.EndOffset - a.BeginOffset; + const lenB = b.EndOffset - b.BeginOffset; + if (lenA !== lenB) { + return lenB - lenA; + } + return b.Score - a.Score; + }); + + // Eliminate overlapping spans deterministically + const nonOverlapping: PiiEntity[] = []; + let lastEnd = -1; + + for (const item of valid) { + if (item.BeginOffset >= lastEnd) { + nonOverlapping.push(item); + lastEnd = item.EndOffset; + } + } + + return nonOverlapping; + } + + private detectLocalFallbackEntities(text: string): PiiEntity[] { + const patterns: Array<{ type: PiiEntityType; regex: RegExp }> = [ + { + type: "EMAIL", + regex: /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g, + }, + { type: "SSN", regex: /\b\d{3}-\d{2}-\d{4}\b/g }, + { + type: "PHONE", + regex: /\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g, + }, + { type: "CREDIT_DEBIT_NUMBER", regex: /\b(?:\d[ -]*?){13,16}\b/g }, + { type: "AWS_ACCESS_KEY", regex: /\bAKIA[0-9A-Z]{16}\b/g }, + { type: "IP_ADDRESS", regex: /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g }, + ]; + + const found: PiiEntity[] = []; + for (const { type, regex } of patterns) { + let match: RegExpExecArray | null; + while ((match = regex.exec(text)) !== null) { + found.push({ + Type: type, + Score: 0.99, + BeginOffset: match.index, + EndOffset: match.index + match[0].length, + }); + } + } + + return found; + } + + private signAwsRequest( + host: string, + payload: string, + amzTarget: string, + ): Record { + const now = new Date(); + const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, ""); + const dateStamp = amzDate.slice(0, 8); + const service = "comprehend"; + const contentType = "application/x-amz-json-1.1"; + const payloadHash = crypto + .createHash("sha256") + .update(payload, "utf8") + .digest("hex"); + + let canonicalHeaders = + `content-type:${contentType}\n` + + `host:${host}\n` + + `x-amz-date:${amzDate}\n` + + `x-amz-target:${amzTarget}\n`; + let signedHeaders = "content-type;host;x-amz-date;x-amz-target"; + + if (this.sessionToken) { + canonicalHeaders += `x-amz-security-token:${this.sessionToken}\n`; + signedHeaders += ";x-amz-security-token"; + } + + const canonicalRequest = [ + "POST", + "/", + "", + canonicalHeaders, + signedHeaders, + payloadHash, + ].join("\n"); + + const algorithm = "AWS4-HMAC-SHA256"; + const credentialScope = `${dateStamp}/${this.region}/${service}/aws4_request`; + const stringToSign = [ + algorithm, + amzDate, + credentialScope, + crypto + .createHash("sha256") + .update(canonicalRequest, "utf8") + .digest("hex"), + ].join("\n"); + + const kDate = crypto + .createHmac("sha256", "AWS4" + this.secretAccessKey) + .update(dateStamp) + .digest(); + const kRegion = crypto + .createHmac("sha256", kDate) + .update(this.region) + .digest(); + const kService = crypto + .createHmac("sha256", kRegion) + .update(service) + .digest(); + const kSigning = crypto + .createHmac("sha256", kService) + .update("aws4_request") + .digest(); + const signature = crypto + .createHmac("sha256", kSigning) + .update(stringToSign, "utf8") + .digest("hex"); + + const authorizationHeader = + `${algorithm} Credential=${this.accessKeyId}/${credentialScope}, ` + + `SignedHeaders=${signedHeaders}, Signature=${signature}`; + + const headers: Record = { + "Content-Type": contentType, + "X-Amz-Date": amzDate, + "X-Amz-Target": amzTarget, + Authorization: authorizationHeader, + }; + if (this.sessionToken) { + headers["X-Amz-Security-Token"] = this.sessionToken; + } + + return headers; + } +} + +export { AwsComprehendRedactor as AwsComprehendRedact }; +export { AwsComprehendRedactor as ComprehendEndpoint }; From 423071209743e98baec056fc03b8cd8725ce4d7f Mon Sep 17 00:00:00 2001 From: wuliangshangren Date: Sat, 7 Nov 2026 08:22:20 +0000 Subject: [PATCH 4/5] test(ai): add comprehensive test suite for AwsComprehendRedactor --- .../src/ai/src/tests/awsComprehend.test.ts | 408 ++++++++++++++++++ 1 file changed, 408 insertions(+) create mode 100644 JS/edgechains/arakoodev/src/ai/src/tests/awsComprehend.test.ts diff --git a/JS/edgechains/arakoodev/src/ai/src/tests/awsComprehend.test.ts b/JS/edgechains/arakoodev/src/ai/src/tests/awsComprehend.test.ts new file mode 100644 index 000000000..bc4a86a11 --- /dev/null +++ b/JS/edgechains/arakoodev/src/ai/src/tests/awsComprehend.test.ts @@ -0,0 +1,408 @@ +import { describe, it, test, expect, beforeEach, vi } from "vitest"; +const jest = vi; +import axios from "axios"; +import { + AwsComprehendRedactor, + AwsComprehendRedact, + ComprehendEndpoint, + ComprehendClientLike, + PiiEntity, + PromptRedactionObservable, +} from "../lib/comprehend/index.js"; + +describe("AwsComprehendRedactor", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe("Injectable Comprehend Client", () => { + test("should use injected client without requiring AWS credentials or network", async () => { + const prompt = "Contact Jane Doe at jane@example.com for assistance."; + const nameStart = prompt.indexOf("Jane Doe"); + const emailStart = prompt.indexOf("jane@example.com"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "NAME", + Score: 0.98, + BeginOffset: nameStart, + EndOffset: nameStart + "Jane Doe".length, + }, + { + Type: "EMAIL", + Score: 0.99, + BeginOffset: emailStart, + EndOffset: emailStart + "jane@example.com".length, + }, + ], + }), + containsPiiEntities: jest.fn().mockResolvedValue({ + Labels: [{ Name: "PII", Score: 0.99 }], + }), + }; + + const redactor = new AwsComprehendRedactor({ + client: mockClient, + redactionMode: "ENTITY_TYPE", + }); + + const result = await redactor.redactText(prompt); + + expect(mockClient.detectPiiEntities).toHaveBeenCalledTimes(1); + expect(result.redactedText).toBe( + "Contact [NAME] at [EMAIL] for assistance.", + ); + expect(result.entities).toHaveLength(2); + expect(result.redactedCount).toBe(2); + + const hasPii = await redactor.containsPiiEntities(prompt); + expect(hasPii).toHaveLength(1); + expect(hasPii[0].Name).toBe("PII"); + }); + }); + + describe("Deterministic Span-based Redaction", () => { + test("should redact multiple non-overlapping spans without offset drift", async () => { + const prompt = + "Customer: John Smith, SSN: 123-45-6789, Call: 555-123-4567 today."; + const nameStart = prompt.indexOf("John Smith"); + const ssnStart = prompt.indexOf("123-45-6789"); + const phoneStart = prompt.indexOf("555-123-4567"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "NAME", + Score: 0.95, + BeginOffset: nameStart, + EndOffset: nameStart + "John Smith".length, + }, + { + Type: "SSN", + Score: 0.99, + BeginOffset: ssnStart, + EndOffset: ssnStart + "123-45-6789".length, + }, + { + Type: "PHONE", + Score: 0.97, + BeginOffset: phoneStart, + EndOffset: phoneStart + "555-123-4567".length, + }, + ], + }), + }; + + const redactor = new AwsComprehendRedactor({ client: mockClient }); + const result = await redactor.redactText(prompt); + + expect(result.redactedText).toBe( + "Customer: [NAME], SSN: [SSN], Call: [PHONE] today.", + ); + }); + + test("should deterministically resolve overlapping spans", async () => { + const text = "Send to: 123 Main Street now."; + const addrStart = text.indexOf("123 Main Street"); + const subStart = text.indexOf("Main Street"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "ADDRESS", + Score: 0.85, + BeginOffset: addrStart, + EndOffset: addrStart + "123 Main Street".length, + }, + { + Type: "NAME", + Score: 0.95, + BeginOffset: subStart, + EndOffset: subStart + "Main Street".length, + }, + ], + }), + }; + + const redactor = new AwsComprehendRedactor({ client: mockClient }); + const result = await redactor.redactText(text); + + expect(result.entities).toHaveLength(1); + expect(result.entities[0].Type).toBe("ADDRESS"); + expect(result.redactedText).toBe("Send to: [ADDRESS] now."); + }); + }); + + describe("Masking Modes & Custom Replacers", () => { + const text = "Email: user@example.com is private."; + const emailStart = text.indexOf("user@example.com"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "EMAIL", + Score: 0.99, + BeginOffset: emailStart, + EndOffset: emailStart + "user@example.com".length, + }, + ], + }), + }; + + test("should mask with MASK character", async () => { + const redactor = new AwsComprehendRedactor({ + client: mockClient, + redactionMode: "MASK", + maskCharacter: "#", + }); + const result = await redactor.redactText(text); + expect(result.redactedText).toBe("Email: ################ is private."); + }); + + test("should support custom replacer function", async () => { + const redactor = new AwsComprehendRedactor({ + client: mockClient, + customReplacer: (entity: PiiEntity, orig: string) => + ``, + }); + const result = await redactor.redactText(text); + expect(result.redactedText).toBe( + "Email: is private.", + ); + }); + }); + + describe("Threshold & Entity Type Filtering", () => { + test("should filter out entities below minScore and excluded entity types", async () => { + const text = "John email: test@example.com tel: 555-123-4567"; + const johnStart = text.indexOf("John"); + const emailStart = text.indexOf("test@example.com"); + const phoneStart = text.indexOf("555-123-4567"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "NAME", + Score: 0.4, + BeginOffset: johnStart, + EndOffset: johnStart + "John".length, + }, + { + Type: "EMAIL", + Score: 0.95, + BeginOffset: emailStart, + EndOffset: emailStart + "test@example.com".length, + }, + { + Type: "PHONE", + Score: 0.99, + BeginOffset: phoneStart, + EndOffset: phoneStart + "555-123-4567".length, + }, + ], + }), + }; + + const redactor = new AwsComprehendRedactor({ + client: mockClient, + minScore: 0.8, + entityTypes: ["EMAIL"], + }); + + const result = await redactor.redactText(text); + + expect(result.redactedText).toBe("John email: [EMAIL] tel: 555-123-4567"); + expect(result.entities).toHaveLength(1); + expect(result.entities[0].Type).toBe("EMAIL"); + }); + }); + + describe("SigV4 Request Generation via Axios", () => { + test("should sign and send SigV4 POST request when AWS credentials provided", async () => { + const text = "SSN: 000-00-0000."; + const ssnStart = text.indexOf("000-00-0000"); + + const targetAxios = (axios as any).default || axios; + const postSpy = vi.spyOn(targetAxios, "post").mockResolvedValueOnce({ + data: { + Entities: [ + { + Type: "SSN", + Score: 0.99, + BeginOffset: ssnStart, + EndOffset: ssnStart + "000-00-0000".length, + }, + ], + }, + }); + + const redactor = new AwsComprehendRedactor({ + region: "us-west-2", + accessKeyId: "AKIAIOSFODNN7EXAMPLE", + secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + sessionToken: "session-token-123", + }); + + const result = await redactor.redactText(text); + + expect(postSpy).toHaveBeenCalledTimes(1); + const [url, body, config] = postSpy.mock.calls[0]; + expect(url).toBe("https://comprehend.us-west-2.amazonaws.com/"); + expect(JSON.parse(body as string)).toEqual({ + Text: text, + LanguageCode: "en", + }); + expect(config?.headers?.["X-Amz-Target"]).toBe( + "Comprehend_20171127.DetectPiiEntities", + ); + expect(config?.headers?.["Authorization"]).toContain("AWS4-HMAC-SHA256"); + expect(config?.headers?.["X-Amz-Security-Token"]).toBe( + "session-token-123", + ); + expect(result.redactedText).toBe("SSN: [SSN]."); + }); + }); + + describe("Local Fallback Rule Redaction", () => { + test("should redact common PII using fallback rules when credentials not set", async () => { + const redactor = new AwsComprehendRedactor({ + fallbackLocalRules: true, + }); + + const text = "Email me at support@edgechains.io or call 123-456-7890."; + const result = await redactor.redactText(text); + + expect(result.redactedText).toContain("[EMAIL]"); + expect(result.redactedText).toContain("[PHONE]"); + }); + }); + + describe("Structured Messages Redaction", () => { + test("should redact array of chat messages", async () => { + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest + .fn() + .mockImplementation(async ({ Text }: { Text: string }) => { + if (Text.includes("secret@company.com")) { + const idx = Text.indexOf("secret@company.com"); + return { + Entities: [ + { + Type: "EMAIL", + Score: 0.99, + BeginOffset: idx, + EndOffset: idx + "secret@company.com".length, + }, + ], + }; + } + return { Entities: [] }; + }), + }; + + const redactor = new AwsComprehendRedactor({ client: mockClient }); + const messages = [ + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "My email is secret@company.com" }, + ]; + + const redacted = await redactor.redactMessages(messages); + expect(redacted[0].content).toBe("You are a helpful assistant."); + expect(redacted[1].content).toBe("My email is [EMAIL]"); + }); + }); + + describe("Observable Chaining with Endpoint Classes", () => { + test("should chain observable directly with endpoint chat method", async () => { + const prompt = "User email: test@example.com"; + const emailStart = prompt.indexOf("test@example.com"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "EMAIL", + Score: 0.99, + BeginOffset: emailStart, + EndOffset: emailStart + "test@example.com".length, + }, + ], + }), + }; + + const redactor = new AwsComprehendRedactor({ client: mockClient }); + const mockEndpoint = { + chat: jest + .fn() + .mockImplementation(async ({ prompt }: { prompt: string }) => { + return { content: `Processed: ${prompt}` }; + }), + }; + + const obs = redactor.chainWithEndpoint(mockEndpoint, { + prompt, + model: "gpt-4o", + }); + + const { redaction, response } = await obs.toPromise(); + + expect(redaction.redactedText).toBe("User email: [EMAIL]"); + expect(mockEndpoint.chat).toHaveBeenCalledWith({ + prompt: "User email: [EMAIL]", + model: "gpt-4o", + }); + expect(response.content).toBe("Processed: User email: [EMAIL]"); + }); + + test("should support observable pipe, map, and subscribe lifecycle", async () => { + const text = "SSN: 111-22-3333"; + const ssnStart = text.indexOf("111-22-3333"); + + const mockClient: ComprehendClientLike = { + detectPiiEntities: jest.fn().mockResolvedValue({ + Entities: [ + { + Type: "SSN", + Score: 0.99, + BeginOffset: ssnStart, + EndOffset: ssnStart + "111-22-3333".length, + }, + ], + }), + }; + + const redactor = new AwsComprehendRedactor({ client: mockClient }); + const obs = redactor.observe(text).map((val) => val.toUpperCase()); + + const nextFn = jest.fn(); + const completeFn = jest.fn(); + + await new Promise((resolve) => { + obs.subscribe({ + next: (val) => nextFn(val), + complete: () => { + completeFn(); + resolve(); + }, + }); + }); + + expect(nextFn).toHaveBeenCalledWith("SSN: [SSN]"); + expect(completeFn).toHaveBeenCalledTimes(1); + }); + }); + + describe("Aliases & Class Exports", () => { + test("should verify class aliases match AwsComprehendRedactor", () => { + expect(AwsComprehendRedact).toBe(AwsComprehendRedactor); + expect(ComprehendEndpoint).toBe(AwsComprehendRedactor); + expect(PromptRedactionObservable).toBeDefined(); + }); + }); +}); From af079e7d8f48159086cb4385bf2693e7c565cad3 Mon Sep 17 00:00:00 2001 From: wuliangshangren Date: Sat, 7 Nov 2026 08:58:39 +0000 Subject: [PATCH 5/5] chore: retrigger checks after CLA signature