diff --git a/.gitleaksignore b/.gitleaksignore index d10cd40a..5a6db07c 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -2,3 +2,4 @@ d7381c5b38cfbc5022685163d3553c4bd73f7815:tests/e2e/fixtures/blocks/000000000000000001bc8a601dd5f0659d36a9b077808850375dfa2d9f009396/meta.json:coinbase-access-token:3 tests/e2e/fixtures/blocks/000000000000000001bc8a601dd5f0659d36a9b077808850375dfa2d9f009396/meta.json:coinbase-access-token:3 tests/e2e/fixtures/blocks/000000000000000001bc8a601dd5f0659d36a9b077808850375dfa2d9f009396/meta.json:coinbase-access-token:10 +tools/fetch-block-fixture/fetch_test.go:coinbase-access-token:30 diff --git a/go.mod b/go.mod index 2175e661..2b6e0fab 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/bsv-blockchain/go-chaincfg v1.7.0 github.com/bsv-blockchain/go-chaintracks v1.2.11 github.com/bsv-blockchain/go-p2p-message-bus v0.1.24 - github.com/bsv-blockchain/go-sdk v1.4.1 - github.com/bsv-blockchain/go-teranode-p2p-client v0.2.9 + github.com/bsv-blockchain/go-sdk v1.5.1 + github.com/bsv-blockchain/go-teranode-p2p-client v0.3.0 github.com/bsv-blockchain/teranode v0.16.0-beta-9 github.com/cockroachdb/pebble v1.1.5 github.com/fergusstrange/embedded-postgres v1.34.0 @@ -19,6 +19,7 @@ require ( github.com/jackc/pgx/v5 v5.11.0 github.com/moby/moby/api v1.56.0 github.com/moby/moby/client v0.6.0 + github.com/mrz1836/go-whatsonchain v1.3.0 github.com/prometheus/client_golang v1.24.1 github.com/prometheus/client_model v0.6.3 github.com/spf13/cobra v1.10.2 @@ -51,38 +52,37 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 // indirect github.com/DataDog/zstd v1.5.8-0.20260421145859-31a7e515a571 // indirect github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 // indirect - github.com/aws/aws-sdk-go-v2 v1.46.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.47.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 // indirect - github.com/aws/aws-sdk-go-v2/config v1.33.3 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.20.3 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.4 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.20.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.112.0 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.9.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.37.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.42.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.49.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.113.1 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 // indirect github.com/aws/smithy-go v1.28.1 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bitcoin-sv/bdk/module/gobdk v1.2.5-0.20260526081552-cdfa7814ee5d // indirect github.com/bsv-blockchain/aerospike-client-go/v8 v8.7.1-bsv5 // indirect github.com/bsv-blockchain/go-batcher/v2 v2.1.0 // indirect - github.com/bsv-blockchain/go-bc v1.2.0 // indirect github.com/bsv-blockchain/go-lockfree-queue v1.2.0 // indirect github.com/bsv-blockchain/go-safe-conversion v1.2.0 // indirect github.com/bsv-blockchain/go-subtree v1.5.0 // indirect github.com/bsv-blockchain/go-tx-map v1.4.1 // indirect github.com/bsv-blockchain/go-wire v1.2.13 // indirect github.com/bytedance/gopkg v0.1.4 // indirect - github.com/bytedance/sonic v1.15.3 // indirect + github.com/bytedance/sonic v1.15.4 // indirect github.com/bytedance/sonic/loader v0.5.2 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect @@ -150,7 +150,6 @@ require ( github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/google/gnostic-models v0.7.1 // indirect github.com/google/gopacket v1.1.19 // indirect - github.com/google/pprof v0.0.0-20260906184651-6331bc6350fe // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.1.0 // indirect @@ -299,7 +298,7 @@ require ( github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/yuin/gopher-lua v1.1.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect - go.mongodb.org/mongo-driver/v2 v2.9.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.9.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.71.0 // indirect go.opentelemetry.io/otel/log v1.47.0-rc.1 // indirect @@ -318,14 +317,14 @@ require ( golang.org/x/net v0.59.0 // indirect golang.org/x/oauth2 v0.37.0 // indirect golang.org/x/sys v0.48.0 // indirect - golang.org/x/telemetry v0.0.0-20260909141559-55ac43c52cc3 // indirect + golang.org/x/telemetry v0.0.0-20260910141331-15ceca2b0a1f // indirect golang.org/x/term v0.46.0 // indirect golang.org/x/text v0.42.0 // indirect golang.org/x/time v0.16.0 // indirect golang.org/x/tools v0.50.0 // indirect gonum.org/v1/gonum v0.17.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260911204522-f61a6ca850bd // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260911204522-f61a6ca850bd // indirect google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect diff --git a/go.sum b/go.sum index 5495f1ca..48d1290b 100644 --- a/go.sum +++ b/go.sum @@ -20,42 +20,42 @@ github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29 h1:0kQAzHq8vL github.com/Microsoft/go-winio v0.6.3-0.20251027160822-ad3df93bed29/go.mod h1:ZWa7ssZJT30CCDGJ7fk/2SBTq9BIQrrVjrcss0UW2s0= github.com/aerospike/aerospike-client-go/v7 v7.10.2 h1:6XK2FeehMg1dninih8BhSuqBN7VUsL/8ExdHR2C6C4c= github.com/aerospike/aerospike-client-go/v7 v7.10.2/go.mod h1:STlBtOkKT8nmp7iD+sEkr/JGEOu+4e2jGlNN0Jiu2a4= -github.com/aws/aws-sdk-go-v2 v1.46.0 h1:1kt7m/EKcEHt5mlyyxx9cSlMddRPIKbjb6DIQsu4HPk= -github.com/aws/aws-sdk-go-v2 v1.46.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= +github.com/aws/aws-sdk-go-v2 v1.47.0 h1:0jsHallhJCeaU0Ko48c/3FK1ctOQ7NpzggxriJOQ8MQ= +github.com/aws/aws-sdk-go-v2 v1.47.0/go.mod h1:bttEH6JqnUL8LepvDVfdrds/fZ5bCIxzpe3abyUrhDU= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20 h1:GPRlPwz40I2B2VrBEASOA3Bi77NyeqejNLkifosX0rs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.20/go.mod h1:g7PNzKcsOKWb4fkSRBA7BZVAS6Y8IcxzN+nRohhQ1Q8= -github.com/aws/aws-sdk-go-v2/config v1.33.3 h1:h090b3O5S17bF87/0ysHZuIT/7DCb4EBRFQX2PMVPCw= -github.com/aws/aws-sdk-go-v2/config v1.33.3/go.mod h1:YYDB1kTejxbfAbEVUqgCtkVp26xvNCHev9cLKABMGAk= -github.com/aws/aws-sdk-go-v2/credentials v1.20.3 h1:tToOYM/LXev4NpfWlIYGDvBvjHmJ3HXpRU9ppl+pM6k= -github.com/aws/aws-sdk-go-v2/credentials v1.20.3/go.mod h1:wfGneWyncO7p67wqXV2IQhPk14JqIc25woKlaArT3WI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2 h1:Ldv7RPHs7qwwTscRjAl3YBud32f3BvdAGRmSvAx5L38= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.19.2/go.mod h1:XyK6UV8xbo66ysVqLd2783C09pBYHOm8aKTRV5DVJ30= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.4 h1:dc/GZeqeKqO/AqxC9FmSIY64wSb6fkFZrxPDONFAOJ8= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.4/go.mod h1:ATGiwbI0OICjQbyVJa++zRFLv4xyZIPCXEnk3NOYbkk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2 h1:q/PSLGuRWCChWg+dLnb9dWOnrCxJtnboXbBtFoqqRrI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.2/go.mod h1:TD1jvU2LvXkJexct5vBqcd8QlNXh5EmRUeL/Z32p0n4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2 h1:6fl86IPqKEXoySqiOWdfgbEp9OVbn44zTfEICNEBDhY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.2/go.mod h1:63HDfhFkdzBpI8WGXTSKUHPKS6mqldj4u3LJW7RZtSU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2 h1:XMgIRS+uW9F3yFKnXGRrI9pkHi99CXTmoz2kz2/TGBA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.2/go.mod h1:vorxDzK+n3jiv9a5ST/LG0Eu9cSv1CRdKTpG6pDMs+M= +github.com/aws/aws-sdk-go-v2/config v1.33.4 h1:FzvkXKSzwqHni4U7nDigHg4jjtqMpVUuHgmZfSoJVQ0= +github.com/aws/aws-sdk-go-v2/config v1.33.4/go.mod h1:VZqGZnZsCWVfK/iGPptJIyNIX3XEX6iQU2Rel4sLrr8= +github.com/aws/aws-sdk-go-v2/credentials v1.20.4 h1:hTvrJJseKbvw32kmiE0G+u/9ZqpqscjDrTigHIXP2qs= +github.com/aws/aws-sdk-go-v2/credentials v1.20.4/go.mod h1:gWp9O1ZBWwpcIrgV+mVHk4gZUurAEDkgypu/OXOlIaw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0 h1:AM4hHjww+PSFtt6E+UrBrPlZkWsePCLEt9AjkfQX+yM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.0/go.mod h1:3x/yXezeQjpOvBb4jEMxrS8SXvpdvJ5abv6l5c1gWM8= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.6 h1:iRNTRmeY62yl77KrpdMNilNEbj2wLfAgoFfMfsK91Go= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.4.6/go.mod h1:Hi9cY8j6kGvrbBZMM66KCtUeGZp9b3VbDSpxPpeDRFI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3 h1:Hp/VgjP0BysR3OgLlR057Vz2LcbbVnoWeJ+3qWiS/fY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.3/go.mod h1:nwGV5qw7F1IZPgxCvA/ph8N2TAuz+BkRG/bXn808qMA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3 h1:MUaM4f+kj1ZIBPZfUS8cxP1GKXXZtHJjAthy93AN7SM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.3/go.mod h1:6YmVmEVRI5ZZzRjCSsb9SryKH0hAlMRdgA7kG9aDvBU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3 h1:fuSCw4Z2qfRCztMPO3GXJNSiEp6Wee+WOLwrHHUMy9c= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.3/go.mod h1:6SxcHheD1pPR5+kWm1wGvjlL/YqUsh267sAfEmN4K7A= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19 h1:bAdDl/HkGCcGPoe25ToSHEw23VIxt6CT5fLcg111BKg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.19/go.mod h1:KaUzbLxv4CeSxh6ZCl9B4m7CuFenS8kUEaDs+f/DQr4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2 h1:CO485zrLOZBeq8mrdsudc8cjyhvC4LY4JFt8ZfHvXZ0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.2/go.mod h1:cBAjVUKBr+5wnx1K3XWS8u5AZsqpA2S/0z7YtI6BGFc= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.2 h1:ZtHYnumr6QyxhzEzNZwzQTFJEXOswrZqTTkRxthwvr4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.2/go.mod h1:a1NXrYpBd311gBzn1UI5UzJyyvXktM4xNh/ydPiPpqY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2 h1:Enx5JcIy/o90kio1LFZ/HguxIsos3clbGuSpGdyshX8= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.2/go.mod h1:Ldmll8/2VqaVerWWDf8IGG+6pAvZY04gs3tevq3nIFs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.112.0 h1:clAH6BH9cZ6JHy6uUfUS26TGuzFSn+gb2lj8DK54TR4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.112.0/go.mod h1:6ZFrmv/qIAYFmG+2WhA7mFU8+edldRlISwT+tNnyLuA= -github.com/aws/aws-sdk-go-v2/service/signin v1.9.0 h1:c3k+k/CS4L+sAIH6fxikL+g5g2LpeNczaoyjjw1iMKI= -github.com/aws/aws-sdk-go-v2/service/signin v1.9.0/go.mod h1:AGIoQg99fBrOIQnF78TLx4lj18mc4gZ0hJx1UaLIFM4= -github.com/aws/aws-sdk-go-v2/service/sso v1.37.0 h1:+rqBaOq7jzInjY8M12hr+zEe85JpRll9BjMx38r33Ok= -github.com/aws/aws-sdk-go-v2/service/sso v1.37.0/go.mod h1:XFlVwUsw3sYh8Hw37umYVJnrcWrwXWRxbNw/JY0bblw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.42.0 h1:hzM3GslEAOBcLn3DHH6ENToFi+vXP+n02W+x6zejAIM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.42.0/go.mod h1:588e7skMkYIYkSUseT8E3WKFfbAfrA1bj3Zf+qxJtJY= -github.com/aws/aws-sdk-go-v2/service/sts v1.49.0 h1:N7Ey8obY3uSui+cxl0OUzFlFmkxSucoJnrniFhw+cLc= -github.com/aws/aws-sdk-go-v2/service/sts v1.49.0/go.mod h1:zMBwjSf4Pt8a1OHYiZ5rPD0PJRK1kQrUaxhS/Dbld8E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3 h1:BHKCSX4QXERe8So8rbWqaM7owqOmDJxATXgJwGng22A= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.3/go.mod h1:GqWeeKfYfezihA2KfFL9l7ohEdZWe1tuFWh3GfyNSnE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3 h1:bON1rJf67TSTDCKg816AAIE4xSTtoo9tl0XRkO72R+I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.3/go.mod h1:c5BBpjJcQXpfeq9iASyVKA3T6vX6B6LEXY4mL/gklDY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3 h1:L8vIOxylma91TcR96NFTEC07G3JDwSl+CvK2b+IODms= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.3/go.mod h1:fmPIZQzTExYuBNWFyi1P7IoDjvskgphXqK1yObMzusM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.113.1 h1:cFHmwLxZPvtoAlo79MboURL2+7b0TMHycnrcf5VZNXk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.113.1/go.mod h1:/uA+2Qj4jd5qBWagVC1AyzzDFXVK997E7U04w7Kw0wI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0 h1:ZD5qFpWcaOKdTuhBi431pIDkCgrMkMlMT6jlpSPoIRI= +github.com/aws/aws-sdk-go-v2/service/signin v1.10.0/go.mod h1:8Nuuf+tR346PjJ3MvZPh9pekbLiLQFWJhzMXfwy7alA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0 h1:JGeeBcMlhg1xtOXYpeCaTQBZObtXMPQCUqBcmr65NRA= +github.com/aws/aws-sdk-go-v2/service/sso v1.38.0/go.mod h1:XwteswG9EOMRFm73UT0t+MbTwyLxMrEXkU6e+v92Lzo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0 h1:obhahQXDEdVEv8y5bTKXR30LVaxYe1kyYM0L7l2Iq+k= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.0/go.mod h1:6twZZ/aXHNy1vXUO8koUbp++MYzMASkOgEBdkbJYmO0= +github.com/aws/aws-sdk-go-v2/service/sts v1.50.0 h1:khXV3+K5D3f4e8xtplaRdSFn1bEg3gj5EBHQvbCOZbQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.50.0/go.mod h1:/8JRcdTt//hG0Q4BTmGbuOplT7ABe+5rdtqUHqXvYIM= github.com/aws/smithy-go v1.28.1 h1:R/nXH00c8qcfCzQVELtRw+eLQWtzv+VAIEFJ1/xxXlQ= github.com/aws/smithy-go v1.28.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -82,12 +82,12 @@ github.com/bsv-blockchain/go-p2p-message-bus v0.1.24 h1:1s6KXWHtG9cBtbF3ZyQlAaa+ github.com/bsv-blockchain/go-p2p-message-bus v0.1.24/go.mod h1:2pvrS51mabnjjzvksuqucd4jQNeqEidg9/OD/iCyifc= github.com/bsv-blockchain/go-safe-conversion v1.2.0 h1:HwLWaxqm2OyU5/2BRQ8kwdn25QyX2HXr0JkfSyTG3DU= github.com/bsv-blockchain/go-safe-conversion v1.2.0/go.mod h1:62Eq2980j4BygfqJhd/ObGRHET8ubEy2rBdGi4k6nsw= -github.com/bsv-blockchain/go-sdk v1.4.1 h1:J2hECn17QkLXq8K8O0m0qA7yih4YmWFTX6o9Ox72Tf8= -github.com/bsv-blockchain/go-sdk v1.4.1/go.mod h1:p2aRf3+hHx8uaQazzVBbX/iFv/vatCFbazrM7RmvT08= +github.com/bsv-blockchain/go-sdk v1.5.1 h1:NGqtUAVjSfwrWPEX2bFNt7yAuwRa0b3G6El4tHZDNHc= +github.com/bsv-blockchain/go-sdk v1.5.1/go.mod h1:clGHSZoOwrnqJrzEw4VHYZk3JQXbrr+mnybCXZCdrt4= github.com/bsv-blockchain/go-subtree v1.5.0 h1:rBp1gqmiDPrsBYS8wr55Jx4EYeXBn7/GSp5ceBJbgvs= github.com/bsv-blockchain/go-subtree v1.5.0/go.mod h1:FgDmkR0YAAsIDn8hUM7hzsg4OeN+EiGtek5RhS7zzmI= -github.com/bsv-blockchain/go-teranode-p2p-client v0.2.9 h1:7l/lm0zbFuTvxF2br261iEd3XN/bYtdIfqewYi/Y1A4= -github.com/bsv-blockchain/go-teranode-p2p-client v0.2.9/go.mod h1:4eIaoM6DjhobU6PE1bqNSbvbf6/M9eEH4SJuv8469mU= +github.com/bsv-blockchain/go-teranode-p2p-client v0.3.0 h1:hT0xqASDX/dQZZjMpZSrNAgbxqIc63DqsDnCJCzdmQg= +github.com/bsv-blockchain/go-teranode-p2p-client v0.3.0/go.mod h1:S+3KgOztBUu5XlgNh+nz/wgNXipLb4nGHkQ2DvU2+NI= github.com/bsv-blockchain/go-tx-map v1.4.1 h1:rY1yuI8O2Ul3ThRBq2l073+oKvLxyXHhKWylZ5G1yf4= github.com/bsv-blockchain/go-tx-map v1.4.1/go.mod h1:0k6O94B3SiWhqu8g8JYWMEHRX1tiHsmKQKi7aATcJuM= github.com/bsv-blockchain/go-wire v1.2.13 h1:g35U+R9zCZWpkTDhi3UyzRnoGsK/uYm9m+FyXY+eiVY= @@ -98,8 +98,8 @@ github.com/bsv-blockchain/testcontainers-aerospike-go v0.4.4 h1:JGK6KYPFV3SjAlKn github.com/bsv-blockchain/testcontainers-aerospike-go v0.4.4/go.mod h1:U6mpzhLam+w9EEJcQtO3avuf+ozSYp2yFSAergkVyUg= github.com/bytedance/gopkg v0.1.4 h1:oZnQwnX82KAIWb7033bEwtxvTqXcYMxDBaQxo5JJHWM= github.com/bytedance/gopkg v0.1.4/go.mod h1:v1zWfPm21Fb+OsyXN2VAHdL6TBb2L88anLQgdyje6R4= -github.com/bytedance/sonic v1.15.3 h1:P3akjLPBtV/i6bHC6LbcLjY3KuoOvfiqF8wFHeP5IhY= -github.com/bytedance/sonic v1.15.3/go.mod h1:8e51yTPdY8M6t+vvGL1c2Y1xL9i+frEeIAQAEl75NUc= +github.com/bytedance/sonic v1.15.4 h1:FgtV/4aBHpla9AxuMpuuzVUpa/Cf3izufkxNmnEzdI8= +github.com/bytedance/sonic v1.15.4/go.mod h1:8e51yTPdY8M6t+vvGL1c2Y1xL9i+frEeIAQAEl75NUc= github.com/bytedance/sonic/loader v0.5.2 h1:0QtP1gevc1OZ6/H8Lb9BRZiCXd1Ftjd3OKuj1T1lBIo= github.com/bytedance/sonic/loader v0.5.2/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/canonical/go-sp800.90a-drbg v0.0.0-20210314144037-6eeb1040d6c3 h1:oe6fCvaEpkhyW3qAicT0TnGtyht/UrgvOwMcEgLb7Aw= @@ -472,6 +472,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWu github.com/mr-tron/base58 v1.1.2/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI= github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8= +github.com/mrz1836/go-whatsonchain v1.3.0 h1:uOq5mHPTMUfiyMAFv+wqRgZXZ5L3CrObaIdShnX/yzU= +github.com/mrz1836/go-whatsonchain v1.3.0/go.mod h1:SAMQltVzWjjBSSIM9KBoLRTFuv9fIrejyEUK95qPDZI= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= @@ -674,8 +676,8 @@ github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.mongodb.org/mongo-driver/v2 v2.9.0 h1:e2mQdOmbkiYz+dj3faM7lVDwl7WdnRD+g5VicafMhL0= -go.mongodb.org/mongo-driver/v2 v2.9.0/go.mod h1:SHKN0IWkKmEVGHLjXnni6s4wPKX4v86FTgOeJJFuXcA= +go.mongodb.org/mongo-driver/v2 v2.9.1 h1:jewiFs2m1/VOQp8qhFshX6hWZ+EAXDhZHXExAUMcOgQ= +go.mongodb.org/mongo-driver/v2 v2.9.1/go.mod h1:SHKN0IWkKmEVGHLjXnni6s4wPKX4v86FTgOeJJFuXcA= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/prometheus v0.71.0 h1:9qgxsFLskbDMXl8WMqThoF6w8yGJgCumn9qRc67OmnI= @@ -794,8 +796,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/telemetry v0.0.0-20260909141559-55ac43c52cc3 h1:Qkx2Wh1A67zgMtn4buvovB7gMx0e15ahZMiXGCJ6rYA= -golang.org/x/telemetry v0.0.0-20260909141559-55ac43c52cc3/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= +golang.org/x/telemetry v0.0.0-20260910141331-15ceca2b0a1f h1:5LLXkapW8RSq5f6l0ztG82Xt1OYp5uRIyMHuZVmkm4k= +golang.org/x/telemetry v0.0.0-20260910141331-15ceca2b0a1f/go.mod h1:i+ivNqjDnTF3WTElsdk5g9V5DTSBYgdNo7xTU9SDwYA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -826,10 +828,10 @@ golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhS golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6 h1:O4Tjo2vlGeGM3+tNgwJrGj5fQHO2V71gDkFqbm328C8= -google.golang.org/genproto/googleapis/api v0.0.0-20260908043556-f8649ddbbfe6/go.mod h1:r4KD2hOq82JBWpTWkJ9NZLf6EwmRvAPIFmk7hPNtd+0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6 h1:ieEbjQ6lzbvntOXUB9nMx9uH+yIU/HbgkNDjnk/mJuk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260908043556-f8649ddbbfe6/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/genproto/googleapis/api v0.0.0-20260911204522-f61a6ca850bd h1:jgCQeWccunvocKNK2RjhQHNiSjSeMMuBpQ/5pbP+2lA= +google.golang.org/genproto/googleapis/api v0.0.0-20260911204522-f61a6ca850bd/go.mod h1:r4KD2hOq82JBWpTWkJ9NZLf6EwmRvAPIFmk7hPNtd+0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260911204522-f61a6ca850bd h1:29LZPNIJg+GEmJwZXruJJRwqy/wnWAYZXDtROToV/aQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260911204522-f61a6ca850bd/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= diff --git a/tools/fetch-block-fixture/fetch.go b/tools/fetch-block-fixture/fetch.go index f9972a29..912f4f6b 100644 --- a/tools/fetch-block-fixture/fetch.go +++ b/tools/fetch-block-fixture/fetch.go @@ -16,6 +16,7 @@ import ( "github.com/bsv-blockchain/go-sdk/chainhash" sdkTx "github.com/bsv-blockchain/go-sdk/transaction" + whatsonchain "github.com/mrz1836/go-whatsonchain" ) // run is the top-level pipeline. Pulled out of main() so a test can @@ -28,6 +29,13 @@ func run(cfg config) error { return fmt.Errorf("mkdir txs: %w", err) } + ctx := context.Background() + + woc, err := newWOCClient(cfg) + if err != nil { + return fmt.Errorf("init whatsonchain client: %w", err) + } + // 1. Fetch block bytes from teranode datahub. teranodeBlock, err := httpGetBytes(cfg.datahubURL+"/block/"+cfg.blockHash, 10*1024*1024) if err != nil { @@ -44,7 +52,7 @@ func run(cfg config) error { // the coinbase txid we extracted from the block binary — // otherwise the WoC and teranode views of the block disagree // and our reconstruction won't match the header root. - txids, err := fetchBlockTxIDs(cfg.wocURL, cfg.blockHash) + txids, err := fetchBlockTxIDs(ctx, woc, cfg.blockHash) if err != nil { return fmt.Errorf("fetch txids: %w", err) } @@ -80,31 +88,16 @@ func run(cfg config) error { // expects when it parses /block/. blockBin := teranodeBlock - // 7. Pick N random non-coinbase txids, fetch each picked tx, then - // enrich it with per-input source script + satoshis pulled from - // each input's parent. The resulting bytes are Extended Format - // (EF) so arcade's intake validator can execute scripts and - // verify fees without needing a chaintracker. Pace fetches at - // one every ~300ms so we stay below WoC's free-tier rate limit - // (retry loop handles sporadic 429s on top of this). + // 7. Pick N random non-coinbase txids, then bulk-fetch each picked tx + // plus every input's parent and build Extended Format (EF) bytes so + // arcade's intake validator can execute scripts and verify fees + // without needing a chaintracker. The WhatsOnChain client's bulk + // processor batches requests and paces them by the configured rate + // limit, and retries sporadic 429s, so no manual throttling is needed. picked := pickRandomTxIDs(txids[1:], cfg.pickN, cfg.rng) - rawTxs := make(map[string][]byte, len(picked)) - parentCache := make(map[string]*sdkTx.Transaction) - first := true - for _, id := range picked { - if !first { - time.Sleep(300 * time.Millisecond) - } - first = false - raw, fErr := fetchRawTx(cfg.wocURL, id) - if fErr != nil { - return fmt.Errorf("fetch raw tx %s: %w", id, fErr) - } - efRaw, fErr := enrichToEF(cfg.wocURL, raw, parentCache) - if fErr != nil { - return fmt.Errorf("enrich tx %s to EF: %w", id, fErr) - } - rawTxs[id] = efRaw + rawTxs, err := buildPickedEFTxs(ctx, woc, picked) + if err != nil { + return err } // 8. Write everything to disk. @@ -293,8 +286,9 @@ func pickRandomTxIDs(txids []string, n int, rng intnRNG) []string { } // httpGetBytes is a small wrapper that GETs a URL with a generous -// timeout, a body cap, and a polite retry on 429 (WhatsOnChain's free -// tier rate-limits when fetching many raw txs in quick succession). +// timeout, a body cap, and a polite retry on 429. It backs the teranode +// datahub fetch; WhatsOnChain calls go through the go-whatsonchain client, +// which handles retries and rate limiting itself. func httpGetBytes(url string, maxBytes int64) ([]byte, error) { c := &http.Client{Timeout: 60 * time.Second} const maxAttempts = 5 @@ -336,92 +330,143 @@ func httpGetBytes(url string, maxBytes int64) ([]byte, error) { return nil, lastErr } -// fetchBlockTxIDs returns the in-block-order txid list (display order) -// for a block. WhatsOnChain caps the inline `tx` field at the first 100 -// txids and surfaces the rest behind pagination links in `pages.uri`. -// We follow every page so blocks of any size return their full txid -// list. -func fetchBlockTxIDs(wocBase, blockHash string) ([]string, error) { - body, err := httpGetBytes(wocBase+"/v1/bsv/main/block/hash/"+blockHash, 32*1024*1024) +// newWOCClient builds a WhatsOnChain client that targets cfg.wocURL. The base +// URL is honored via WithBaseURL so --woc can still point at a mirror, proxy, +// or local test server. The client retries 429/5xx/network errors with +// exponential backoff and paces its bulk processors by the rate limit, which +// replaces the tool's former manual 429 loop and ~300ms sleeps. +func newWOCClient(cfg config) (whatsonchain.ClientInterface, error) { + base := strings.TrimRight(cfg.wocURL, "/") + "/v1/" + return whatsonchain.NewClient(context.Background(), + whatsonchain.WithBaseURL(base), + whatsonchain.WithRateLimit(3), + whatsonchain.WithRequestTimeout(60*time.Second), + whatsonchain.WithRequestRetryCount(4), + whatsonchain.WithBackoff(time.Second, 16*time.Second, 2.0, 250*time.Millisecond), + whatsonchain.WithUserAgent("arcade-fetch-block-fixture"), + ) +} + +// fetchBlockTxIDs returns the in-block-order txid list (display order) for a +// block. GetBlockByHash returns the first page of txids inline in Tx and, for +// large blocks, the remaining pages as URIs in Pages; we fetch each additional +// page (1-based) and concatenate so blocks of any size return their full txid +// list. The caller cross-checks the length and coinbase against the parsed +// block, which guards this inline-first-page + numbered-pages ordering. +func fetchBlockTxIDs(ctx context.Context, woc whatsonchain.ClientInterface, blockHash string) ([]string, error) { + info, err := woc.GetBlockByHash(ctx, blockHash) if err != nil { return nil, err } - var resp struct { - Tx []string `json:"tx"` - Pages struct { - URI []string `json:"uri"` - } `json:"pages"` - } - if err := json.Unmarshal(body, &resp); err != nil { - return nil, fmt.Errorf("decode block JSON: %w", err) - } - if len(resp.Tx) == 0 { + if len(info.Tx) == 0 { return nil, fmt.Errorf("WhatsOnChain returned no txids for block %s", blockHash) } - all := resp.Tx - for _, uri := range resp.Pages.URI { - // Page URIs in the response are server-relative - // ("/block/hash//page/") and don't include the - // /v1/bsv/main prefix; rebuild against wocBase. - pageURL := wocBase + "/v1/bsv/main" + uri - pageBody, err := httpGetBytes(pageURL, 32*1024*1024) - if err != nil { - return nil, fmt.Errorf("fetch page %s: %w", uri, err) + all := info.Tx + for i := range info.Pages.URI { + page, pErr := woc.GetBlockPages(ctx, blockHash, i+1) + if pErr != nil { + return nil, fmt.Errorf("fetch page %d: %w", i+1, pErr) } - var pageTxs []string - if err := json.Unmarshal(pageBody, &pageTxs); err != nil { - return nil, fmt.Errorf("decode page %s: %w", uri, err) - } - all = append(all, pageTxs...) + all = append(all, page...) } return all, nil } -// fetchRawTx returns the raw tx bytes for a txid. WhatsOnChain serves -// hex at /v1/bsv/main/tx//hex. -func fetchRawTx(wocBase, txid string) ([]byte, error) { - body, err := httpGetBytes(wocBase+"/v1/bsv/main/tx/"+txid+"/hex", 4*1024*1024) +// bulkRawTxs fetches raw tx bytes for each txid via WhatsOnChain's bulk raw +// transaction endpoint, keyed by txid. The SDK's processor batches the ids +// (20 per request) and paces the batches by the client's rate limit. It +// returns an error if any requested txid is missing from the response. +func bulkRawTxs(ctx context.Context, woc whatsonchain.ClientInterface, ids []string) (map[string][]byte, error) { + out := make(map[string][]byte, len(ids)) + if len(ids) == 0 { + return out, nil + } + list, err := woc.BulkRawTransactionDataProcessor(ctx, &whatsonchain.TxHashes{TxIDs: ids}) if err != nil { return nil, err } - hexStr := strings.TrimSpace(string(body)) - raw, err := hex.DecodeString(hexStr) - if err != nil { - preview := hexStr - if len(preview) > 64 { - preview = preview[:64] + for _, info := range list { + if info == nil { + continue + } + raw, dErr := hex.DecodeString(strings.TrimSpace(info.Hex)) + if dErr != nil { + return nil, fmt.Errorf("decode hex for tx %s: %w", info.TxID, dErr) + } + out[info.TxID] = raw + } + for _, id := range ids { + if _, ok := out[id]; !ok { + return nil, fmt.Errorf("WhatsOnChain did not return tx %s", id) } - return nil, fmt.Errorf("decode hex: %w (preview=%q)", err, preview) } - return raw, nil + return out, nil } -// enrichToEF takes canonical raw tx bytes, fetches each input's parent -// tx from WoC (using the parentCache to deduplicate shared parents -// across picked txs), injects the source script + satoshis into each -// input via SetSourceTxOutput, and returns the Extended Format bytes. -// Pace between WoC calls is ~300ms to stay under the free-tier rate -// limit, same cadence as the picked-tx loop. -func enrichToEF(wocBase string, raw []byte, parentCache map[string]*sdkTx.Transaction) ([]byte, error) { - tx, err := sdkTx.NewTransactionFromBytes(raw) +// buildPickedEFTxs bulk-fetches the picked txs and all of their input parents, +// then builds Extended Format (EF) bytes for each picked tx. Parent txids are +// de-duplicated and sorted so batching (and therefore output) is deterministic +// for a fixed pick, preserving the tool's fixed-seed idempotency. +func buildPickedEFTxs(ctx context.Context, woc whatsonchain.ClientInterface, picked []string) (map[string][]byte, error) { + // Phase 1: bulk-fetch and parse the picked txs, collecting parent txids. + pickedRaw, err := bulkRawTxs(ctx, woc, picked) if err != nil { - return nil, fmt.Errorf("parse picked tx: %w", err) + return nil, fmt.Errorf("fetch picked txs: %w", err) + } + pickedTxs := make(map[string]*sdkTx.Transaction, len(picked)) + parentSet := make(map[string]struct{}) + for _, id := range picked { + tx, pErr := sdkTx.NewTransactionFromBytes(pickedRaw[id]) + if pErr != nil { + return nil, fmt.Errorf("parse picked tx %s: %w", id, pErr) + } + pickedTxs[id] = tx + for _, input := range tx.Inputs { + parentSet[input.SourceTXID.String()] = struct{}{} + } + } + + // Phase 2: bulk-fetch and parse the unique parents (sorted for determinism). + parentIDs := make([]string, 0, len(parentSet)) + for id := range parentSet { + parentIDs = append(parentIDs, id) + } + sort.Strings(parentIDs) + parentRaw, err := bulkRawTxs(ctx, woc, parentIDs) + if err != nil { + return nil, fmt.Errorf("fetch parent txs: %w", err) + } + parentCache := make(map[string]*sdkTx.Transaction, len(parentIDs)) + for _, id := range parentIDs { + p, pErr := sdkTx.NewTransactionFromBytes(parentRaw[id]) + if pErr != nil { + return nil, fmt.Errorf("parse parent tx %s: %w", id, pErr) + } + parentCache[id] = p } + + // Phase 3: build EF bytes for each picked tx (no network). + rawTxs := make(map[string][]byte, len(picked)) + for _, id := range picked { + efRaw, eErr := enrichToEF(pickedTxs[id], parentCache) + if eErr != nil { + return nil, fmt.Errorf("enrich tx %s to EF: %w", id, eErr) + } + rawTxs[id] = efRaw + } + return rawTxs, nil +} + +// enrichToEF injects each input's source script + satoshis (taken from the +// input's parent tx in parentCache) into tx via SetSourceTxOutput, then returns +// the Extended Format (EF) bytes. Every parent referenced by tx's inputs must +// be present in parentCache. +func enrichToEF(tx *sdkTx.Transaction, parentCache map[string]*sdkTx.Transaction) ([]byte, error) { for vin, input := range tx.Inputs { parentTXID := input.SourceTXID.String() parent, ok := parentCache[parentTXID] if !ok { - time.Sleep(300 * time.Millisecond) - parentRaw, fErr := fetchRawTx(wocBase, parentTXID) - if fErr != nil { - return nil, fmt.Errorf("input %d: fetch parent %s: %w", vin, parentTXID, fErr) - } - p, pErr := sdkTx.NewTransactionFromBytes(parentRaw) - if pErr != nil { - return nil, fmt.Errorf("input %d: parse parent %s: %w", vin, parentTXID, pErr) - } - parentCache[parentTXID] = p - parent = p + return nil, fmt.Errorf("input %d: missing parent %s", vin, parentTXID) } if int(input.SourceTxOutIndex) >= len(parent.Outputs) { return nil, fmt.Errorf("input %d: parent %s has %d outputs, vout=%d", diff --git a/tools/fetch-block-fixture/fetch_test.go b/tools/fetch-block-fixture/fetch_test.go new file mode 100644 index 00000000..e6ea71b8 --- /dev/null +++ b/tools/fetch-block-fixture/fetch_test.go @@ -0,0 +1,478 @@ +package main + +import ( + "context" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand/v2" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/bsv-blockchain/go-sdk/script" + sdkTx "github.com/bsv-blockchain/go-sdk/transaction" + whatsonchain "github.com/mrz1836/go-whatsonchain" +) + +// committedBlockHash is the mainnet block the e2e fixture was generated from. +// tests/e2e/fixtures/blocks//block.bin is a real teranode block binary +// (txCount=1911, single subtree) we reuse to drive run() against real bytes. +const committedBlockHash = "000000000000000001bc8a601dd5f0659d36a9b077808850375dfa2d9f009396" + +const ( + committedTxCount = 1911 + committedCoinbaseTx = "30e099b7318973c83020132c6b1b77f0a6bade9a4d28d36c004076d8a674d7c6" + committedBlockBinRel = "../../tests/e2e/fixtures/blocks/" + committedBlockHash + "/block.bin" +) + +// wocServer is a configurable httptest stand-in for the WhatsOnChain REST API. +// It serves the three endpoints the fixture tool uses: block-by-hash, block +// pages, and bulk raw transactions (POST /txs/hex). +type wocServer struct { + *httptest.Server + + blockInline []string // inline Tx from GetBlockByHash + blockPages [][]string // additional pages; index i => page (i+1) + rawByTxID map[string]string // txid -> raw tx hex, for /txs/hex + bulkCalls atomic.Int32 // number of POST /txs/hex requests served +} + +func newWOCServer(t *testing.T) *wocServer { + t.Helper() + ws := &wocServer{rawByTxID: map[string]string{}} + ws.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case r.Method == http.MethodGet && strings.Contains(path, "/block/hash/") && strings.Contains(path, "/page/"): + parts := strings.Split(path, "/page/") + n, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil || n-1 < 0 || n-1 >= len(ws.blockPages) { + http.Error(w, "no such page", http.StatusNotFound) + return + } + _ = json.NewEncoder(w).Encode(ws.blockPages[n-1]) + case r.Method == http.MethodGet && strings.Contains(path, "/block/hash/"): + resp := map[string]any{"tx": ws.blockInline} + if len(ws.blockPages) > 0 { + uris := make([]string, len(ws.blockPages)) + for i := range ws.blockPages { + uris[i] = fmt.Sprintf("/block/hash/x/page/%d", i+1) + } + resp["pages"] = map[string]any{"uri": uris} + } + _ = json.NewEncoder(w).Encode(resp) + case r.Method == http.MethodPost && strings.HasSuffix(path, "/txs/hex"): + ws.bulkCalls.Add(1) + var req struct { + TxIDs []string `json:"txids"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + out := make([]map[string]string, 0, len(req.TxIDs)) + for _, id := range req.TxIDs { + if h, ok := ws.rawByTxID[id]; ok { + out = append(out, map[string]string{"txid": id, "hex": h}) + } + } + _ = json.NewEncoder(w).Encode(out) + default: + http.Error(w, "unexpected path: "+path, http.StatusNotFound) + } + })) + t.Cleanup(ws.Close) + return ws +} + +// testClient builds a WhatsOnChain client pointed at baseURL with a high rate +// limit and no retries so unit tests stay fast. +func testClient(t *testing.T, baseURL string) whatsonchain.ClientInterface { + t.Helper() + c, err := whatsonchain.NewClient(context.Background(), + whatsonchain.WithBaseURL(strings.TrimRight(baseURL, "/")+"/v1/"), + whatsonchain.WithRateLimit(1000), + whatsonchain.WithRequestRetryCount(0), + ) + if err != nil { + t.Fatalf("new woc client: %v", err) + } + return c +} + +func mustScript(t *testing.T, hexStr string) *script.Script { + t.Helper() + s, err := script.NewFromHex(hexStr) + if err != nil { + t.Fatalf("script from hex: %v", err) + } + return s +} + +// p2pkh returns a distinct throwaway locking script keyed by fill byte. +func p2pkh(t *testing.T, fill byte) *script.Script { + t.Helper() + return mustScript(t, "76a914"+strings.Repeat(fmt.Sprintf("%02x", fill), 20)+"88ac") +} + +// TestFetchBlockTxIDs_SinglePage verifies the inline tx list is returned when a +// block has no additional pages. +func TestFetchBlockTxIDs_SinglePage(t *testing.T) { + ws := newWOCServer(t) + ws.blockInline = []string{"a", "b", "c"} + + got, err := fetchBlockTxIDs(context.Background(), testClient(t, ws.URL), "hash") + if err != nil { + t.Fatalf("fetchBlockTxIDs: %v", err) + } + if want := []string{"a", "b", "c"}; !equalStrings(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +// TestFetchBlockTxIDs_Pagination verifies inline txids are concatenated with +// every advertised page, in order. +func TestFetchBlockTxIDs_Pagination(t *testing.T) { + ws := newWOCServer(t) + ws.blockInline = []string{"a", "b"} + ws.blockPages = [][]string{{"c", "d"}, {"e"}} + + got, err := fetchBlockTxIDs(context.Background(), testClient(t, ws.URL), "hash") + if err != nil { + t.Fatalf("fetchBlockTxIDs: %v", err) + } + if want := []string{"a", "b", "c", "d", "e"}; !equalStrings(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} + +// TestFetchBlockTxIDs_Empty verifies an empty tx list is treated as an error. +func TestFetchBlockTxIDs_Empty(t *testing.T) { + ws := newWOCServer(t) + ws.blockInline = nil + + _, err := fetchBlockTxIDs(context.Background(), testClient(t, ws.URL), "hash") + if err == nil { + t.Fatal("expected error for empty tx list, got nil") + } +} + +// TestBulkRawTxs_Batching verifies more than one batch is issued for >20 ids and +// that every id is returned, hex-decoded. +func TestBulkRawTxs_Batching(t *testing.T) { + ws := newWOCServer(t) + ids := make([]string, 25) + for i := range ids { + id := fmt.Sprintf("%064x", i) + ids[i] = id + ws.rawByTxID[id] = fmt.Sprintf("%02x00", i%256) // arbitrary valid hex + } + + got, err := bulkRawTxs(context.Background(), testClient(t, ws.URL), ids) + if err != nil { + t.Fatalf("bulkRawTxs: %v", err) + } + if len(got) != len(ids) { + t.Fatalf("got %d txs, want %d", len(got), len(ids)) + } + if calls := ws.bulkCalls.Load(); calls != 2 { + t.Fatalf("expected 2 batches for 25 ids (max 20/batch), got %d", calls) + } + wantFirst, _ := hex.DecodeString(ws.rawByTxID[ids[0]]) + if string(got[ids[0]]) != string(wantFirst) { + t.Fatalf("decoded bytes mismatch for %s", ids[0]) + } +} + +// TestBulkRawTxs_MissingTx verifies a requested id absent from the response is +// an error. +func TestBulkRawTxs_MissingTx(t *testing.T) { + ws := newWOCServer(t) + ws.rawByTxID["a"] = "0100" + + _, err := bulkRawTxs(context.Background(), testClient(t, ws.URL), []string{"a", "b"}) + if err == nil || !strings.Contains(err.Error(), "did not return tx b") { + t.Fatalf("expected missing-tx error, got %v", err) + } +} + +// TestBulkRawTxs_BadHex verifies a non-hex payload is surfaced as an error. +func TestBulkRawTxs_BadHex(t *testing.T) { + ws := newWOCServer(t) + ws.rawByTxID["a"] = "zzzz" + + _, err := bulkRawTxs(context.Background(), testClient(t, ws.URL), []string{"a"}) + if err == nil || !strings.Contains(err.Error(), "decode hex") { + t.Fatalf("expected decode-hex error, got %v", err) + } +} + +// TestBulkRawTxs_Empty verifies no request is made for an empty id list. +func TestBulkRawTxs_Empty(t *testing.T) { + ws := newWOCServer(t) + got, err := bulkRawTxs(context.Background(), testClient(t, ws.URL), nil) + if err != nil { + t.Fatalf("bulkRawTxs(empty): %v", err) + } + if len(got) != 0 { + t.Fatalf("expected empty map, got %d", len(got)) + } + if calls := ws.bulkCalls.Load(); calls != 0 { + t.Fatalf("expected 0 batches for empty input, got %d", calls) + } +} + +// TestEnrichToEF verifies the source script + satoshis are injected from the +// parent and that EF bytes are produced. +func TestEnrichToEF(t *testing.T) { + lock0 := p2pkh(t, 0x00) + parent := sdkTx.NewTransaction() + parent.AddOutput(&sdkTx.TransactionOutput{Satoshis: 1000, LockingScript: lock0}) + + child := sdkTx.NewTransaction() + child.AddInputFromTx(parent, 0, nil) + child.AddOutput(&sdkTx.TransactionOutput{Satoshis: 900, LockingScript: p2pkh(t, 0x22)}) + + // Re-parse from canonical bytes so the input carries no source output, + // mirroring what WhatsOnChain serves. + parsed, err := sdkTx.NewTransactionFromBytes(child.Bytes()) + if err != nil { + t.Fatalf("parse child: %v", err) + } + cache := map[string]*sdkTx.Transaction{parent.TxID().String(): parent} + + ef, err := enrichToEF(parsed, cache) + if err != nil { + t.Fatalf("enrichToEF: %v", err) + } + if len(ef) == 0 { + t.Fatal("expected non-empty EF bytes") + } + in := parsed.Inputs[0] + if in.SourceTxSatoshis() == nil || *in.SourceTxSatoshis() != 1000 { + t.Fatalf("source satoshis = %v, want 1000", in.SourceTxSatoshis()) + } + if in.SourceTxScript() == nil || in.SourceTxScript().String() != lock0.String() { + t.Fatalf("source script = %v, want %s", in.SourceTxScript(), lock0.String()) + } +} + +// TestEnrichToEF_MissingParent verifies a missing parent is an error. +func TestEnrichToEF_MissingParent(t *testing.T) { + parent := sdkTx.NewTransaction() + parent.AddOutput(&sdkTx.TransactionOutput{Satoshis: 1000, LockingScript: p2pkh(t, 0x00)}) + child := sdkTx.NewTransaction() + child.AddInputFromTx(parent, 0, nil) + parsed, err := sdkTx.NewTransactionFromBytes(child.Bytes()) + if err != nil { + t.Fatalf("parse child: %v", err) + } + + _, err = enrichToEF(parsed, map[string]*sdkTx.Transaction{}) + if err == nil || !strings.Contains(err.Error(), "missing parent") { + t.Fatalf("expected missing-parent error, got %v", err) + } +} + +// TestEnrichToEF_VoutOutOfRange verifies a vout beyond the parent's outputs is +// an error. +func TestEnrichToEF_VoutOutOfRange(t *testing.T) { + parent := sdkTx.NewTransaction() + parent.AddOutput(&sdkTx.TransactionOutput{Satoshis: 1000, LockingScript: p2pkh(t, 0x00)}) + child := sdkTx.NewTransaction() + child.AddInputFromTx(parent, 0, nil) + parsed, err := sdkTx.NewTransactionFromBytes(child.Bytes()) + if err != nil { + t.Fatalf("parse child: %v", err) + } + parsed.Inputs[0].SourceTxOutIndex = 5 // parent has only 1 output + + _, err = enrichToEF(parsed, map[string]*sdkTx.Transaction{parent.TxID().String(): parent}) + if err == nil || !strings.Contains(err.Error(), "vout=5") { + t.Fatalf("expected vout-out-of-range error, got %v", err) + } +} + +// TestBuildPickedEFTxs verifies the two-phase bulk flow: picked txs and their +// (de-duplicated) parents are fetched, EF bytes are built, and output is +// deterministic for a fixed pick. +func TestBuildPickedEFTxs(t *testing.T) { + // One parent with two outputs, spent by two different children. + parent := sdkTx.NewTransaction() + parent.AddOutput(&sdkTx.TransactionOutput{Satoshis: 1000, LockingScript: p2pkh(t, 0x01)}) + parent.AddOutput(&sdkTx.TransactionOutput{Satoshis: 2000, LockingScript: p2pkh(t, 0x02)}) + + child1 := sdkTx.NewTransaction() + child1.AddInputFromTx(parent, 0, nil) + child1.AddOutput(&sdkTx.TransactionOutput{Satoshis: 900, LockingScript: p2pkh(t, 0x03)}) + + child2 := sdkTx.NewTransaction() + child2.AddInputFromTx(parent, 1, nil) + child2.AddOutput(&sdkTx.TransactionOutput{Satoshis: 1900, LockingScript: p2pkh(t, 0x04)}) + + ws := newWOCServer(t) + for _, tx := range []*sdkTx.Transaction{parent, child1, child2} { + ws.rawByTxID[tx.TxID().String()] = hex.EncodeToString(tx.Bytes()) + } + picked := sortedStrings(child1.TxID().String(), child2.TxID().String()) + + got, err := buildPickedEFTxs(context.Background(), testClient(t, ws.URL), picked) + if err != nil { + t.Fatalf("buildPickedEFTxs: %v", err) + } + if len(got) != 2 { + t.Fatalf("got %d EF txs, want 2", len(got)) + } + for _, id := range picked { + if len(got[id]) == 0 { + t.Fatalf("missing EF bytes for %s", id) + } + } + // One batch for the picked txs + one batch for the single shared parent. + if calls := ws.bulkCalls.Load(); calls != 2 { + t.Fatalf("expected 2 bulk batches (picked + deduped parents), got %d", calls) + } + + // Determinism: a second identical run yields byte-identical EF output. + got2, err := buildPickedEFTxs(context.Background(), testClient(t, ws.URL), picked) + if err != nil { + t.Fatalf("buildPickedEFTxs (2nd): %v", err) + } + for _, id := range picked { + if string(got[id]) != string(got2[id]) { + t.Fatalf("non-deterministic EF output for %s", id) + } + } +} + +// Test429Retry verifies the WhatsOnChain client retries a 429 and then +// succeeds, confirming retry is wired through the fixture tool's call path. +func Test429Retry(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if calls.Add(1) == 1 { + w.WriteHeader(http.StatusTooManyRequests) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"tx": []string{"a", "b"}}) + })) + defer srv.Close() + + client, err := whatsonchain.NewClient(context.Background(), + whatsonchain.WithBaseURL(srv.URL+"/v1/"), + whatsonchain.WithRequestRetryCount(2), + whatsonchain.WithBackoff(time.Millisecond, 5*time.Millisecond, 2.0, 0), + ) + if err != nil { + t.Fatalf("new client: %v", err) + } + + got, err := fetchBlockTxIDs(context.Background(), client, "hash") + if err != nil { + t.Fatalf("fetchBlockTxIDs: %v", err) + } + if want := []string{"a", "b"}; !equalStrings(got, want) { + t.Fatalf("got %v, want %v", got, want) + } + if calls.Load() != 2 { + t.Fatalf("expected 2 attempts (429 then 200), got %d", calls.Load()) + } +} + +// TestNewWOCClient_BaseURL verifies --woc is wired through WithBaseURL. +func TestNewWOCClient_BaseURL(t *testing.T) { + client, err := newWOCClient(config{wocURL: "https://example.com"}) + if err != nil { + t.Fatalf("newWOCClient: %v", err) + } + if got, want := client.BaseURL(), "https://example.com/v1/"; got != want { + t.Fatalf("BaseURL() = %q, want %q", got, want) + } +} + +// TestRun_TxidCountMismatch drives run() with the real committed block binary +// (served by a datahub stand-in) and a WhatsOnChain stand-in whose txid list +// disagrees with the block's txCount, asserting the guard fires. +func TestRun_TxidCountMismatch(t *testing.T) { + datahub := newDatahubServer(t) + ws := newWOCServer(t) + ws.blockInline = []string{committedCoinbaseTx} // len 1 != committedTxCount + + err := run(runConfig(t, datahub.URL, ws.URL)) + if err == nil || !strings.Contains(err.Error(), "txid count mismatch") { + t.Fatalf("expected txid count mismatch, got %v", err) + } +} + +// TestRun_CoinbaseMismatch drives run() with a WhatsOnChain stand-in that +// returns the right number of txids but a wrong coinbase at index 0. +func TestRun_CoinbaseMismatch(t *testing.T) { + datahub := newDatahubServer(t) + ws := newWOCServer(t) + txids := make([]string, committedTxCount) + for i := range txids { + txids[i] = fmt.Sprintf("%064x", i) // none equal the real coinbase + } + ws.blockInline = txids + + err := run(runConfig(t, datahub.URL, ws.URL)) + if err == nil || !strings.Contains(err.Error(), "coinbase txid mismatch") { + t.Fatalf("expected coinbase txid mismatch, got %v", err) + } +} + +// newDatahubServer serves the committed block binary at /block/, mirroring +// the teranode datahub the tool fetches from. +func newDatahubServer(t *testing.T) *httptest.Server { + t.Helper() + blockBin, err := os.ReadFile(committedBlockBinRel) + if err != nil { + t.Fatalf("read committed block.bin: %v", err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.Contains(r.URL.Path, "/block/") { + http.Error(w, "unexpected path", http.StatusNotFound) + return + } + _, _ = w.Write(blockBin) + })) + t.Cleanup(srv.Close) + return srv +} + +func runConfig(t *testing.T, datahubURL, wocURL string) config { + t.Helper() + return config{ + blockHash: committedBlockHash, + outDir: t.TempDir(), + datahubURL: datahubURL, + wocURL: wocURL, + pickN: 1, + rng: rand.New(rand.NewPCG(1, 1)), //nolint:gosec // deterministic test seed + } +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func sortedStrings(vals ...string) []string { + out := append([]string(nil), vals...) + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j-1] > out[j]; j-- { + out[j-1], out[j] = out[j], out[j-1] + } + } + return out +}