fix: improve cctp v2 offchain lookup service

.changeset/fuzzy-yaks-pull.md

@@ -0,0 +1,6 @@
+---
+"@hyperlane-xyz/ccip-server": patch
+"@hyperlane-xyz/sdk": patch
+---
+
+Improve CCTP offchain lookup server error handling

typescript/ccip-server/src/services/CCTPAttestationService.ts

@@ -6,9 +6,21 @@ import {
   UnhandledErrorReason,
 } from '../utils/prometheus.js';
 
+// https://developers.circle.com/api-reference/cctp/all/get-messages-v-2
+type DelayReason =
+  | 'insufficient_fee'
+  | 'amount_above_max'
+  | 'insufficient_allowance_available';
+type Status = 'complete' | 'pending_confirmations';
+
 interface CCTPMessageEntry {
   attestation: string;
   message: string;
+  eventNonce: string;
+  // CCTP v2 only
+  cctpVersion?: string;
+  status?: Status;
+  delayReason?: DelayReason;
 }
 
 interface CCTPData {
@@ -170,8 +182,60 @@ class CCTPAttestationService {
       throw new Error(`CCTP attestation request failed: ${resp.statusText}`);
     }
 
-    const json: CCTPData = await resp.json();
+    let json: CCTPData;
+    try {
+      json = await resp.json();
+    } catch (error) {
+      logger.error(
+        {
+          ...context,
+          status: resp.status,
+          statusText: resp.statusText,
+          url,
+          messageId,
+          error_reason:
+            UnhandledErrorReason.CCTP_ATTESTATION_SERVICE_JSON_PARSE_ERROR,
+        },
+        'CCTP attestation response parsing failed',
+      );
+      throw new Error(`CCTP service response parsing failed: ${error}`);
+    }
+
+    json.messages.forEach((message) => {
+      if (message.attestation === 'PENDING') {
+        const errorString = 'CCTP attestation is pending';
+        switch (message.delayReason) {
+          case 'insufficient_fee':
+          case 'amount_above_max':
+          case 'insufficient_allowance_available':
+            PrometheusMetrics.logUnhandledError(
+              this.serviceName,
+              UnhandledErrorReason.CCTP_ATTESTATION_SERVICE_PENDING,
+            );
+            logger.error(
+              {
+                error_reason:
+                  UnhandledErrorReason.CCTP_ATTESTATION_SERVICE_PENDING,
+                ...message,
+                ...context,
+              },
+              errorString + ` due to ${message.delayReason}`,
+            );
+            break;
+          default:
+            logger.info(
+              {
+                ...context,
+                ...message,
+              },
+              errorString,
+            );
+        }
+        throw new Error(errorString);
+      }
+    });
 
+    // TODO: handle multiple messages in one tx hash
     return [json.messages[0].message, json.messages[0].attestation];
   }
 }

typescript/ccip-server/src/utils/prometheus.ts

@@ -27,6 +27,8 @@ export enum UnhandledErrorReason {
   CCTP_UNSUPPORTED_VERSION = 'cctp_unsupported_version',
   CCTP_ATTESTATION_SERVICE_500 = 'cctp_attestation_service_500',
   CCTP_ATTESTATION_SERVICE_UNKNOWN_ERROR = 'cctp_attestation_service_unknown_error',
+  CCTP_ATTESTATION_SERVICE_JSON_PARSE_ERROR = 'cctp_attestation_service_json_parse_error',
+  CCTP_ATTESTATION_SERVICE_PENDING = 'cctp_attestation_service_pending',
 
   // CallCommitments errors
   CALL_COMMITMENTS_DATABASE_ERROR = 'call_commitments_database_error',

typescript/sdk/src/ism/metadata/ccipread.ts

@@ -1,7 +1,7 @@
 import { utils } from 'ethers';
 
 import { AbstractCcipReadIsm__factory } from '@hyperlane-xyz/core';
-import { WithAddress } from '@hyperlane-xyz/utils';
+import { WithAddress, ensure0x } from '@hyperlane-xyz/utils';
 
 import { HyperlaneCore } from '../../core/HyperlaneCore.js';
 import { IsmType, OffchainLookupIsmConfig } from '../types.js';
@@ -58,11 +58,11 @@ export class OffchainLookupMetadataBuilder implements MetadataBuilder {
       const url = urlTemplate
         .replace('{sender}', sender)
         .replace('{data}', callDataHex);
+
+      let res: Response;
       try {
-        let responseJson: any;
         if (urlTemplate.includes('{data}')) {
-          const res = await fetch(url);
-          responseJson = await res.json();
+          res = await fetch(url);
         } else {
           const signature = await signer.signMessage(
             utils.arrayify(
@@ -73,20 +73,30 @@ export class OffchainLookupMetadataBuilder implements MetadataBuilder {
               ),
             ),
           );
-          const res = await fetch(url, {
+          res = await fetch(url, {
             method: 'POST',
             headers: { 'Content-Type': 'application/json' },
             body: JSON.stringify({ sender, data: callDataHex, signature }),
           });
-          responseJson = await res.json();
         }
-        const rawHex = responseJson.data as string;
-        return rawHex.startsWith('0x') ? rawHex : `0x${rawHex}`;
       } catch (error: any) {
         this.core.logger.warn(
           `CCIP-read metadata fetch failed for ${url}: ${error}`,
         );
         // try next URL
+        continue;
+      }
+
+      try {
+        const responseJson = await res.json();
+        if (res.ok) {
+          return ensure0x(responseJson.data);
+        }
+      } catch (error) {
+        this.core.logger.warn(
+          `CCIP-read metadata fetch failed for ${url}: ${error}`,
+        );
+        // try next URL
       }
     }