Skip to content
This repository was archived by the owner on Oct 14, 2026. It is now read-only.

Commit 532455c

Browse files
jindaliiitaclaude
andcommitted
fix(tokowaka-client): merge WAF classification into checkEdgeOptimizeStatus, fix Imperva false-positive
- checkEdgeOptimizeStatus now classifies the same response it already fetches for a WAF/bot-block, so edgeOptimizeEnabled reflects both routing AND WAF status (routing alone is not sufficient); adds blocked/statusCode to the response. - Removes '_incapsula_resource' from BOT_CHALLENGE_KEYWORDS, matching the same fix already shipped in spacecat-import-worker's vendored copy — Imperva/Incapsula injects this into every page it fronts, blocked or not, causing real pages to be misclassified as blocked. - Introduces one shared ADOBE_PROBE_USER_AGENT constant used by both checkEdgeOptimizeStatus and checkWafConnectivity, so the two checks can no longer drift into sending different UAs. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
1 parent 1d89ab3 commit 532455c

4 files changed

Lines changed: 194 additions & 42 deletions

File tree

‎packages/spacecat-shared-tokowaka-client/src/index.js‎

Lines changed: 29 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import {
4545
EDGE_OPTIMIZE_PROXY_BASE_URL_DEFAULT,
4646
PRIVATE_HOST_RE,
4747
WAF_PROBE_TIMEOUT_MS,
48+
ADOBE_PROBE_USER_AGENT,
4849
classifyProbeResponse,
4950
} from './utils/waf-probe-utils.js';
5051

@@ -1339,11 +1340,18 @@ class TokowakaClient {
13391340
}
13401341

13411342
/**
1342-
* Checks if Edge Optimize is enabled for a specific page path
1343-
* Follows one level of redirect and retries on failures
1343+
* Checks if Edge Optimize is enabled for a specific page path. Retries on failures.
1344+
*
1345+
* Fetches the customer's own domain directly (no proxy) and checks the response for
1346+
* BOTH the routing signal (x-tokowaka-request-id / x-edgeoptimize-request-id header)
1347+
* and a WAF/bot-block (via classifyProbeResponse, on the same fetched response — no
1348+
* second request). `edgeOptimizeEnabled` folds both together: it is only `true` when
1349+
* the request-id header is present AND the response isn't classified as blocked —
1350+
* routing alone is not sufficient.
1351+
*
13441352
* @param {Object} site - Site entity
13451353
* @param {string} path - Path to check (e.g., '/products/chair')
1346-
* @returns {Promise<Object>} - Status result with edgeOptimizeEnabled flag
1354+
* @returns {Promise<Object>} - `{ edgeOptimizeEnabled, blocked, statusCode? }`
13471355
*/
13481356
async checkEdgeOptimizeStatus(site, path) {
13491357
if (!isNonEmptyObject(site)) {
@@ -1362,15 +1370,15 @@ class TokowakaClient {
13621370
}
13631371

13641372
const defaultProbeHeaders = {
1365-
// eslint-disable-next-line max-len
1366-
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Tokowaka-AI Tokowaka/1.0 AdobeEdgeOptimize-AI AdobeEdgeOptimize/1.0',
1373+
'User-Agent': ADOBE_PROBE_USER_AGENT,
13671374
'fastly-debug': '1',
13681375
};
13691376
const customHeaders = currentConfig?.getScraperConfig?.()?.headers ?? {};
13701377
const probeHeaders = { ...customHeaders, ...defaultProbeHeaders };
13711378

13721379
const baseURL = getEffectiveBaseURL(site);
13731380
const targetUrl = new URL(path, baseURL).toString();
1381+
const targetHost = new URL(targetUrl).host;
13741382

13751383
this.log.info(`[edge-optimize-status] Checking edge optimize status for ${targetUrl}`);
13761384

@@ -1393,21 +1401,30 @@ class TokowakaClient {
13931401

13941402
this.log.info(`[edge-optimize-status] Response status: ${response.status} for ${targetUrl}`);
13951403

1396-
const edgeOptimizeEnabled = response.headers.get('x-tokowaka-request-id') !== null
1404+
const requestIdPresent = response.headers.get('x-tokowaka-request-id') !== null
13971405
|| response.headers.get('x-edgeoptimize-request-id') !== null;
13981406

1399-
this.log.info(`[edge-optimize-status] Edge optimize headers found: ${edgeOptimizeEnabled} for ${targetUrl}`);
1407+
this.log.info(`[edge-optimize-status] Edge optimize headers found: ${requestIdPresent} for ${targetUrl}`);
1408+
1409+
// Classify the same response already fetched above for a WAF/bot-block — no second
1410+
// request. edgeOptimizeEnabled folds both signals together: routing alone isn't
1411+
// enough if the customer's own WAF is blocking us.
1412+
// eslint-disable-next-line no-await-in-loop
1413+
const { blocked, statusCode } = await classifyProbeResponse(response, targetHost, this.log);
1414+
const edgeOptimizeEnabled = requestIdPresent && !blocked;
14001415

14011416
return {
14021417
edgeOptimizeEnabled,
1418+
blocked,
1419+
statusCode,
14031420
};
14041421
} catch (error) {
14051422
const isTimeout = error?.code === 'ETIMEOUT';
14061423

14071424
if (isTimeout) {
14081425
// eslint-disable-next-line max-len
14091426
this.log.warn(`[edge-optimize-status] Request timed out after ${REQUEST_TIMEOUT_MS}ms for ${targetUrl}, returning edgeOptimizeEnabled: false`);
1410-
return { edgeOptimizeEnabled: false };
1427+
return { edgeOptimizeEnabled: false, blocked: null };
14111428
}
14121429

14131430
attempt += 1;
@@ -1442,8 +1459,9 @@ class TokowakaClient {
14421459
}
14431460

14441461
/**
1445-
* Probes whether a WAF or Bot Manager is blocking AdobeEdgeOptimize/1.0 traffic
1446-
* for the site.
1462+
* Probes whether a WAF or Bot Manager is blocking Adobe Edge Optimize traffic for the
1463+
* site, via Adobe's own edge-optimize proxy (not a direct fetch to the customer — see
1464+
* checkEdgeOptimizeStatus for the direct-fetch WAF check used alongside routing status).
14471465
*
14481466
* Probe outcomes:
14491467
* - Hard block: HTTP 401/403/406/429/503 → `{ reachable: false, blocked: true }`
@@ -1476,7 +1494,7 @@ class TokowakaClient {
14761494

14771495
const response = await tracingFetch(EDGE_OPTIMIZE_PROXY_BASE_URL_DEFAULT, {
14781496
method: 'GET',
1479-
headers: { 'x-forwarded-host': targetHost },
1497+
headers: { 'x-forwarded-host': targetHost, 'User-Agent': ADOBE_PROBE_USER_AGENT },
14801498
signal: AbortSignal.timeout(WAF_PROBE_TIMEOUT_MS),
14811499
});
14821500

‎packages/spacecat-shared-tokowaka-client/src/utils/waf-probe-utils.js‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ export const PRIVATE_HOST_RE = /^(localhost$|127\.|10\.|192\.168\.|172\.(1[6-9]|
1717

1818
export const WAF_PROBE_TIMEOUT_MS = 15000;
1919

20+
// Single, shared, Adobe-identifying UA for every Edge Optimize probe (routing check and
21+
// WAF check alike) so a customer only ever needs one allowlist rule to cover all of them,
22+
// and so the two checks can never drift into sending different UAs again.
23+
export const ADOBE_PROBE_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) '
24+
+ 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 '
25+
+ 'Tokowaka-AI AdobeEdgeOptimize-AI Spacecat/1.0';
26+
2027
// Soft-block detection: vendor-specific technical identifiers that only appear in
2128
// WAF-generated challenge pages, never in real page content. Broad natural-language
2229
// terms ('challenge', 'captcha', 'access denied') are intentionally excluded — they
@@ -25,7 +32,11 @@ export const WAF_PROBE_TIMEOUT_MS = 15000;
2532
export const BOT_CHALLENGE_KEYWORDS = [
2633
'cf-chl-widget', // Cloudflare challenge widget CSS class
2734
'completing the challenge', // Cloudflare-specific challenge phrase
28-
'_incapsula_resource', // Imperva/Incapsula JS artifact — only in WAF-generated pages
35+
// '_incapsula_resource' removed: Imperva/Incapsula injects this fingerprinting/RASP
36+
// script into every page it fronts, blocked or not — false-positived real pages
37+
// (e.g. hyundaicanada.com's genuine homepage). Imperva hard blocks are still caught
38+
// via HARD_BLOCK_STATUS_CODES (403/429/etc.). Same fix already applied to the
39+
// vendored copy in spacecat-import-worker's waf-probe.js.
2940
'errors.edgesuite.net', // Akamai error page domain
3041
'errors.edgekey.net', // Akamai edge key domain
3142
];
@@ -36,14 +47,17 @@ export const BOT_CHALLENGE_KEYWORDS = [
3647
export const HARD_BLOCK_STATUS_CODES = new Set([401, 403, 406, 429, 503]);
3748

3849
/**
39-
* Classifies an already-fetched Tokowaka-proxied response into one of four probe outcomes:
50+
* Classifies an already-fetched response into one of four probe outcomes. Used both for
51+
* responses fetched through the Tokowaka proxy (checkWafConnectivity) and for responses
52+
* fetched directly from the customer's own domain (checkEdgeOptimizeStatus) — the
53+
* classification logic itself doesn't care which vantage point the response came from:
4054
* - Hard block : HTTP status in HARD_BLOCK_STATUS_CODES → { reachable: false, blocked: true }
4155
* - CF challenge: cf-mitigated: challenge header → { reachable: false, blocked: true }
4256
* - Soft block : 2xx HTML body with vendor keywords → { reachable: false, blocked: true }
4357
* - Clean pass : 2xx with real content → { reachable: true, blocked: false }
4458
* - Other : unexpected status (e.g. redirect) → { reachable: false, blocked: false }
4559
*
46-
* @param {Response} response - Fetch response from the Tokowaka proxy.
60+
* @param {Response} response - Already-fetched response to classify.
4761
* @param {string} targetHost - Customer hostname, used only for log messages.
4862
* @param {Object} log - Logger with an `info` method.
4963
* @returns {Promise<Object>} Classification result.

0 commit comments

Comments
 (0)