Vercel Integration: Log Pipeline & Metrics corrections & improvements

vercel/assets/logs/vercel.yaml

@@ -38,13 +38,76 @@ facets:
     name: Subdivision Name
     path: network.client.geoip.subdivision.name
     source: log
+  - groups:
+      - Vercel
+    name: Vercel Environment
+    path: environment
+    source: log
+    : string
+  - groups:
+      - Vercel
+    name: Vercel Deployment Id
+    path: deploymentId
+    source: log
+    : string
+  - groups:
+      - Vercel
+    name: Vercel Invocation Id
+    path: invocationId
+    source: log
+    : string
+  - groups:
+      - Vercel
+    name: Vercel Project Id
+    path: projectId
+    source: log
+    : string
+  - groups:
+      - Vercel
+    name: Vercel Project Name
+    path: projectName
+    source: log
+    type: string
+  - groups:
+      - Vercel
+    name: Vercel Request Duration
+    path: lambda.duration
+    source: log
+    type: integer
+    unit:
+      family: time
+      name: nanosecond
 pipeline:
   type: pipeline
   name: Vercel
   enabled: true
   filter:
     query: source:vercel
   processors:
+    - type: pipeline
+      name: Processing for URL Paths
+      enabled: true
+      filter:
+        query: "@proxy.scheme:* @proxy.host:* @proxy.path:* NOT @source:build"
+      processors:
+        - type: string-builder-processor
+          name: Build a parseable http.url from proxy scheme, host, and path
+          enabled: true
+          template: '%{proxy.scheme}://%{proxy.host}%{proxy.path}'
+          target: http.url
+          replaceMissing: true
+        - type: url-parser
+          name: Parse http.url into http.url_details
+          enabled: true
+          sources:
+            - http.url
+          target: http.url_details
+        - type: url-parser
+          name: Parse proxy.referer into http.referer_details
+          enabled: true
+          sources:
+            - proxy.referer
+          target: http.referer_details
     - type: grok-parser
       name: Parsing Lambda Report Logs
       enabled: true
@@ -64,17 +127,20 @@ pipeline:
       name: Status Code Remapper
       enabled: true
       categories:
+        - filter:
+            query: '@type:stdout AND @source:build'
+          name: Info
         - filter:
             query: '@proxy.statusCode:[200 TO 299]'
           name: OK
         - filter:
             query: '@proxy.statusCode:[300 TO 399]'
           name: Notice
         - filter:
-            query: '@proxy.statusCode:[400 TO 499]'
+            query: '@proxy.statusCode:[400 TO 499] OR (@type:stderr AND @source:build) OR message:*warn*'
           name: Warning
         - filter:
-            query: '@proxy.statusCode:[500 TO 599]'
+            query: '@proxy.statusCode:[500 TO 599] OR (@type:stderr AND NOT @source:build AND NOT message:*warn*)'
           name: Error
       target: http.status_category
     - type: status-remapper
@@ -87,6 +153,26 @@ pipeline:
       enabled: true
       sources:
         - projectName
+    - type: attribute-remapper
+      name: Define `deploymentId` as the official version of the log
+      enabled: true
+      sources:
+        - deploymentId
+      sourceType: attribute
+      target: version
+      targetType: tag
+      preserveSource: false
+      overrideOnConflict: false
+    - type: attribute-remapper
+      name: Define `environment` as the official env of the log
+      enabled: true
+      sources:
+        - environment
+      sourceType: attribute
+      target: env
+      targetType: tag
+      preserveSource: true
+      overrideOnConflict: false
     - type: attribute-remapper
       name: Map `host` to `url_host`
       enabled: true
@@ -131,6 +217,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -158,6 +246,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -185,6 +275,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -215,6 +307,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -245,6 +339,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -275,6 +371,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -305,6 +403,8 @@ metrics:
         tagName: branch
       - path: '@environment'
         tagName: environment
+      - path: '@deploymentId'
+        tagName: deployment_id
       - path: '@projectId'
         tagName: project_id
       - path: '@statusCode'
@@ -324,7 +424,11 @@ metrics:
   - name: vercel.projects.build_count
     query: source:vercel "Build completed in"
     groupBys:
+      - path: '@environment'
+        tagName: environment
       - path: '@projectName'
         tagName: project_name
       - path: '@projectId'
         tagName: project_id
+      - path: '@deploymentId'
+        tagName: deployment_id