From 0b2354c0357fb13910ef36d111c7055344811a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Mon, 8 Sep 2025 11:01:44 +0200 Subject: [PATCH 1/3] fix: call addPoints if filtering is disabled --- src/filtering.ts | 65 +++++++++++++++++++++++++----------------------- src/proxy.ts | 41 +++++++++++++++++------------- 2 files changed, 58 insertions(+), 48 deletions(-) diff --git a/src/filtering.ts b/src/filtering.ts index c2a0b29..8da2bf2 100644 --- a/src/filtering.ts +++ b/src/filtering.ts @@ -6,20 +6,26 @@ export type AIResponse = { reason: string } -export interface Message { - text: string - messageId?: string - threadId?: string - parent?: string - flagged?: boolean - reason?: string +export declare enum MessageType { + TEXT = 'text', + THREAD = 'thread', + REACTION = 'reaction', } -export interface UserMessage { - message: Message - timestamp: number +export interface MessageData { + id: string + type: MessageType + message: string username: string - address?: string + address: string + timestamp: number + index: string + topic: string + targetMessageId?: string + signature?: string + flagged?: boolean + reason?: string + isLegacy?: boolean } export async function callAI( @@ -85,21 +91,18 @@ export async function callAI( return retV } -function addPoint(userMessage: UserMessage) { +export function addPoint(userMessage: MessageData) { const { DEVCON_BACKEND_URL, DEVCON_BACKEND_API_KEY } = process.env as EnvironmentVariables if (DEVCON_BACKEND_URL && DEVCON_BACKEND_API_KEY) { - // only chat messages have address - if (!userMessage.address) { - fetch(DEVCON_BACKEND_URL + '/addpoints/' + userMessage.username, { - method: 'POST', - headers: { - Authorization: `Bearer ${DEVCON_BACKEND_API_KEY}`, - }, - }).catch((error: any) => { - logger.error('Error calling backend', error) - }) - } + fetch(DEVCON_BACKEND_URL + '/addpoints/' + userMessage.username, { + method: 'POST', + headers: { + Authorization: `Bearer ${DEVCON_BACKEND_API_KEY}`, + }, + }).catch((error: any) => { + logger.error('Error calling backend', error) + }) } } @@ -112,27 +115,27 @@ export async function doFiltering( threshold: number, ): Promise { const bodyBuffer = Buffer.from(body) - const userMessage = JSON.parse(bodyBuffer.toString('utf8')) as UserMessage + const userMessage = JSON.parse(bodyBuffer.toString('utf8')) as MessageData if (typeof userMessage.message === 'string') { - userMessage.message = JSON.parse(userMessage.message) as Message + userMessage.message = JSON.parse(userMessage.message) as string } let aiResponse: AIResponse = { flagged: false, reason: '' } - if (userMessage.message.text.trim().length >= threshold) { - aiResponse = await callAI(prompt, userMessage.message.text, APIUrl, key, timeout) - userMessage.message.flagged = aiResponse.flagged + if (userMessage.message.trim().length >= threshold) { + aiResponse = await callAI(prompt, userMessage.message, APIUrl, key, timeout) + userMessage.flagged = aiResponse.flagged if (!aiResponse.flagged) { addPoint(userMessage) } } else { - userMessage.message.flagged = false - logger.info(`skipped: ${userMessage.message.text} - too short`) + userMessage.flagged = false + logger.info(`skipped: ${userMessage.message} - too short`) } if (aiResponse.flagged) { - logger.info(`flagged: ${userMessage.message.text} - ${aiResponse.reason}`) + logger.info(`flagged: ${userMessage.message} - ${aiResponse.reason}`) } return Buffer.from(JSON.stringify(userMessage)) diff --git a/src/proxy.ts b/src/proxy.ts index 261306d..48ed10c 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -6,7 +6,7 @@ import { subdomainToBzz } from './bzz-link' import { logger } from './logger' import { StampsManager } from './stamps' import { getErrorMessage } from './utils' -import { doFiltering } from './filtering' +import { addPoint, doFiltering, MessageData } from './filtering' export const GET_PROXY_ENDPOINTS = ['/chunks/*', '/bytes/*', '/bzz/*', '/feeds/*'] export const POST_PROXY_ENDPOINTS = ['/chunks', '/bytes', '/bzz', '/soc/*', '/feeds/*'] @@ -77,24 +77,31 @@ export function createProxyEndpoints(app: Application, options: Options) { await fetchAndRespond('GET', req.path, req.query, req.headers, req.body, res, options) }) app.post(POST_PROXY_ENDPOINTS, async (req, res) => { - if (options.filteringActive && req.path.startsWith('/bytes') && req.method === 'POST') { - const originalBody = req.body - try { - req.body = await doFiltering( - Buffer.from(req.body), - options.filteringPrompt, - options.filteringUrl, - options.filteringKey, - options.filteringTimeout, - options.filteringThreshold, - ) - const bodySize = Buffer.byteLength(req.body) - req.headers['content-length'] = bodySize.toString() - } catch (error) { - req.body = originalBody - logger.error(`proxy failed: ${getErrorMessage(error)}`) + if (req.path.startsWith('/bytes') && req.method === 'POST') { + if (options.filteringActive) { + const originalBody = req.body + try { + req.body = await doFiltering( + Buffer.from(req.body), + options.filteringPrompt, + options.filteringUrl, + options.filteringKey, + options.filteringTimeout, + options.filteringThreshold, + ) + const bodySize = Buffer.byteLength(req.body) + req.headers['content-length'] = bodySize.toString() + } catch (error) { + req.body = originalBody + logger.error(`proxy failed: ${getErrorMessage(error)}`) + } + } else { + const bodyBuffer = Buffer.from(req.body) + const userMessage = JSON.parse(bodyBuffer.toString('utf8')) as MessageData + addPoint(userMessage) } } + await fetchAndRespond('POST', req.path, req.query, req.headers, req.body, res, options) }) } From 195ee68aa4756e1e50f6423509a07d2b99daf424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Mon, 8 Sep 2025 11:26:30 +0200 Subject: [PATCH 2/3] chore: update workflow step versions --- .github/workflows/check.yaml | 6 +++--- .github/workflows/tests.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check.yaml b/.github/workflows/check.yaml index 489fb52..edb64aa 100644 --- a/.github/workflows/check.yaml +++ b/.github/workflows/check.yaml @@ -17,16 +17,16 @@ jobs: node-version: [14.x] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} ## Try getting the node modules from cache, if failed npm ci - - uses: actions/cache@v2 + - uses: actions/cache@v4 id: cache-npm with: path: node_modules diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d68d74d..7b022e8 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,12 +21,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 1 - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} @@ -34,7 +34,7 @@ jobs: run: | echo "${{ secrets.GITHUB_TOKEN }}" | docker login https://docker.pkg.github.com -u ${GITHUB_ACTOR} --password-stdin ## Try getting the node modules from cache, if failed npm ci - - uses: actions/cache@v2 + - uses: actions/cache@v4 id: cache-npm with: path: node_modules From 009c78cfa979a1a86dd6ba064bf35028d999e139 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Ujv=C3=A1ri?= Date: Mon, 8 Sep 2025 11:56:41 +0200 Subject: [PATCH 3/3] fix: optional filtering Options --- .github/workflows/configs_sync.yaml | 4 ++-- .github/workflows/publish_deb.yaml | 4 ++-- .github/workflows/publish_docker.yaml | 2 +- .github/workflows/publish_npmjs.yaml | 4 ++-- src/config.ts | 12 ++++++------ src/proxy.ts | 23 ++++++++++++----------- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/.github/workflows/configs_sync.yaml b/.github/workflows/configs_sync.yaml index 374bc8b..757ea79 100644 --- a/.github/workflows/configs_sync.yaml +++ b/.github/workflows/configs_sync.yaml @@ -2,7 +2,7 @@ name: Sync configuration files on: schedule: - - cron: "0 0 * * *" + - cron: '0 0 * * *' push: branches: - 'master' @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Check and update the Bee version uses: ethersphere/repo-sync-action@v1 diff --git a/.github/workflows/publish_deb.yaml b/.github/workflows/publish_deb.yaml index b83ca6c..e7b728c 100644 --- a/.github/workflows/publish_deb.yaml +++ b/.github/workflows/publish_deb.yaml @@ -8,8 +8,8 @@ jobs: docker-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 14 registry-url: 'https://registry.npmjs.org' diff --git a/.github/workflows/publish_docker.yaml b/.github/workflows/publish_docker.yaml index b591949..d1b8caf 100644 --- a/.github/workflows/publish_docker.yaml +++ b/.github/workflows/publish_docker.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Login to Docker and Quay diff --git a/.github/workflows/publish_npmjs.yaml b/.github/workflows/publish_npmjs.yaml index 7dbc95c..dcff551 100644 --- a/.github/workflows/publish_npmjs.yaml +++ b/.github/workflows/publish_npmjs.yaml @@ -9,8 +9,8 @@ jobs: publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v1 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: 14 registry-url: 'https://registry.npmjs.org' diff --git a/src/config.ts b/src/config.ts index 7a12050..82755c4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -11,12 +11,12 @@ export interface AppConfig { exposeHashedIdentity?: boolean readinessCheck?: boolean homepage?: string - filteringActive: boolean - filteringKey: string - filteringUrl: string - filteringPrompt: string - filteringTimeout: number - filteringThreshold: number + filteringActive?: boolean + filteringKey?: string + filteringUrl?: string + filteringPrompt?: string + filteringTimeout?: number + filteringThreshold?: number } export interface ServerConfig { diff --git a/src/proxy.ts b/src/proxy.ts index 48ed10c..2a6f553 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -7,6 +7,7 @@ import { logger } from './logger' import { StampsManager } from './stamps' import { getErrorMessage } from './utils' import { addPoint, doFiltering, MessageData } from './filtering' +import { DEFAULT_FILTERING_THRESHOLD } from './config' export const GET_PROXY_ENDPOINTS = ['/chunks/*', '/bytes/*', '/bzz/*', '/feeds/*'] export const POST_PROXY_ENDPOINTS = ['/chunks', '/bytes', '/bzz', '/soc/*', '/feeds/*'] @@ -28,12 +29,12 @@ interface Options { ensSubdomains?: boolean remap: Record userAgents?: string[] - filteringActive: boolean - filteringKey: string - filteringUrl: string - filteringPrompt: string - filteringTimeout: number - filteringThreshold: number + filteringActive?: boolean + filteringKey?: string + filteringUrl?: string + filteringPrompt?: string + filteringTimeout?: number + filteringThreshold?: number } export function createProxyEndpoints(app: Application, options: Options) { @@ -83,11 +84,11 @@ export function createProxyEndpoints(app: Application, options: Options) { try { req.body = await doFiltering( Buffer.from(req.body), - options.filteringPrompt, - options.filteringUrl, - options.filteringKey, - options.filteringTimeout, - options.filteringThreshold, + options.filteringPrompt || '', + options.filteringUrl || '', + options.filteringKey || '', + options.filteringTimeout || 3000, + options.filteringThreshold || DEFAULT_FILTERING_THRESHOLD, ) const bodySize = Buffer.byteLength(req.body) req.headers['content-length'] = bodySize.toString()