Repository navigation
Expand file tree
/
Copy pathaction.yml
More file actions
156 lines (146 loc) · 5.94 KB
/
Copy pathaction.yml
File metadata and controls
156 lines (146 loc) · 5.94 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
name: spantruth
description: Check the values an LLM instrumentation library reports against what the provider sent.
author: Plexito
branding:
icon: check-circle
color: green
inputs:
subject:
description: >-
A built-in subject, such as openllmetry, or the path of a subject directory in your
repository, such as ./spantruth.
required: true
with:
description: >-
Packages to replace in the subject, one per line: a local checkout such as
./packages/my-instrumentation, a git ref, or a version.
required: false
default: ""
scenario:
description: Scenario IDs to run, one per line. Every scenario when empty.
required: false
default: ""
expect:
description: >-
The path of an expectation file. The step fails when the failing checks differ from it.
Write one with `spantruth run --write-expect FILE`.
required: false
default: ""
latest:
description: Unpin the subject's tracked packages and the provider SDKs.
required: false
default: "false"
artifact-name:
description: The name of the artifact with the results and the HTML report.
required: false
default: ""
comment:
description: >-
On a pull request, keep one comment up to date with what changed. It needs
`pull-requests: write`. A fork's pull request has a read-only token, so the step warns and
the job goes on.
required: false
default: "false"
token:
description: The token that writes the comment.
required: false
default: ${{ github.token }}
outputs:
exit-code:
description: >-
0 when every cell ran and the failing checks match the expectation file, 1 when they
differ, 2 for a usage error, 3 when spantruth could not run a cell.
value: ${{ steps.run.outputs.exit-code }}
results:
description: The path of the results document.
value: ${{ steps.run.outputs.results }}
runs:
using: composite
steps:
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
with:
enable-cache: true
cache-suffix: spantruth
# Inputs reach the script through the environment only, never through ${{ }} in the script.
# spantruth is installed from the checkout of this action, so its version is the action ref.
- id: run
shell: bash
env:
SUBJECT: ${{ inputs.subject }}
WITH: ${{ inputs.with }}
SCENARIO: ${{ inputs.scenario }}
EXPECT: ${{ inputs.expect }}
LATEST: ${{ inputs.latest }}
ARTIFACT_NAME: ${{ inputs.artifact-name }}
COMMENT: ${{ inputs.comment }}
run: |
set -euo pipefail
# When installing spantruth fails, the run could not happen: report exit code 3.
trap 'echo "exit-code=3" >> "$GITHUB_OUTPUT"' ERR
# One directory per step: a job can run the action more than once.
out="$(mktemp -d "$RUNNER_TEMP/spantruth.XXXXXX")"
uv venv --quiet --no-config --python 3.12 "$out/venv"
uv pip install --quiet --no-config --python "$out/venv/bin/python" "$GITHUB_ACTION_PATH"
spantruth="$out/venv/bin/spantruth"
args=(run --subject "$SUBJECT" --out "$out/results.json" --summary "$GITHUB_STEP_SUMMARY")
while read -r line; do
if [ -n "$line" ]; then args+=(--with "$line"); fi
done <<< "$WITH"
while read -r line; do
if [ -n "$line" ]; then args+=(--scenario "$line"); fi
done <<< "$SCENARIO"
if [ -n "$EXPECT" ]; then args+=(--expect "$EXPECT"); fi
if [ "$LATEST" = "true" ]; then args+=(--latest); fi
if [ "$COMMENT" = "true" ]; then
args+=(--comment "$out/comment.md")
args+=(--comment-link "${GITHUB_SERVER_URL:-}/${GITHUB_REPOSITORY:-}/actions/runs/${GITHUB_RUN_ID:-}")
fi
code=0
"$spantruth" "${args[@]}" || code=$?
if [ -f "$out/results.json" ]; then
"$spantruth" report "$out/results.json" --html "$out/report.html" > /dev/null ||
echo "::warning::spantruth could not render the HTML report"
fi
name="${ARTIFACT_NAME:-spantruth-$(basename "$SUBJECT")}"
{
echo "exit-code=$code"
echo "results=$out/results.json"
echo "out-dir=$out"
echo "artifact-name=$name"
if [ -f "$out/comment.md" ]; then
echo "comment-file=$out/comment.md"
echo "comment-marker=$(head -n 1 "$out/comment.md")"
fi
} >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
if: always() && steps.run.outputs.artifact-name != ''
with:
name: ${{ steps.run.outputs.artifact-name }}
path: |
${{ steps.run.outputs.out-dir }}/results.json
${{ steps.run.outputs.out-dir }}/report.html
if-no-files-found: ignore
# A comment is a convenience: nothing here can fail the job.
- id: comment
if: always() && github.event.pull_request.number && steps.run.outputs.comment-file != ''
shell: bash
env:
GH_TOKEN: ${{ inputs.token }}
PR_NUMBER: ${{ github.event.pull_request.number }}
REPOSITORY: ${{ github.repository }}
COMMENT_FILE: ${{ steps.run.outputs.comment-file }}
MARKER: ${{ steps.run.outputs.comment-marker }}
run: |
comments="repos/$REPOSITORY/issues/$PR_NUMBER/comments"
id="$(gh api --paginate "$comments" --jq '.[] | select(.body | startswith(env.MARKER)) | .id' | head -n 1)"
if [ -n "$id" ] && gh api -X PATCH "repos/$REPOSITORY/issues/comments/$id" -F "body=@$COMMENT_FILE" > /dev/null; then
exit 0
fi
if ! gh api -X POST "$comments" -F "body=@$COMMENT_FILE" > /dev/null; then
echo "::warning::spantruth could not comment. The token needs pull-requests: write, and a pull request from a fork has a read-only token."
fi
exit 0
- shell: bash
env:
CODE: ${{ steps.run.outputs.exit-code }}
run: exit "${CODE:-3}"