diff --git a/README.md b/README.md index 7a8fa8b..f0a996e 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,39 @@ func main() { For fully working examples on how to use templates, non-blocking processing and more, take a look at [`examples/`](https://github.com/transloadit/go-sdk/tree/main/examples). +## Assembly Notifications (webhooks) + +If an assembly's `NotifyURL` is set, Transloadit submits the assembly's final status to that +URL once it finishes. Use `ParseAssemblyNotification` in your HTTP handler to verify the +request's signature and decode the payload: + +```go +http.HandleFunc("/webhook", func(w http.ResponseWriter, r *http.Request) { + notification, err := transloadit.ParseAssemblyNotification(r, options.AuthSecret) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // notification.Ok / notification.Error describe the assembly's outcome. + // notification.AssemblyInfo has the same fields as GetAssembly's result. +}) +``` + +See [`examples/webhook`](https://github.com/transloadit/go-sdk/tree/main/examples/webhook) for a +full example. + +## Bearer tokens + +`Client.IssueBearerToken` exchanges your Auth Key and Auth Secret for a short-lived, scoped +bearer token: + +```go +token, err := client.IssueBearerToken(context.Background(), transloadit.BearerTokenRequest{ + Scope: "assemblies:read assemblies:write", +}) +``` + ## Documentation See Godoc for full API documentation. diff --git a/assembly.go b/assembly.go index bc7ce92..6a7b049 100644 --- a/assembly.go +++ b/assembly.go @@ -115,9 +115,97 @@ type AssemblyInfo struct { // Since 7 March 2018, the user agent, IP and referer are no longer // stored by Transloadit (see https://transloadit.com/blog/2018/03/gdpr/) // Therefore, these properties will always hold empty strings. - ClientAgent string - ClientIp string - ClientReferer string + ClientAgent string `json:"client_agent"` + ClientIp string `json:"client_ip"` + ClientReferer string `json:"client_referer"` + + // The following fields were added to match the Assembly Status schema + // published at https://api2.transloadit.com/openapi.json, which is also + // used for the notify_url webhook payload (see AssemblyNotificationPayload). + AccountID string `json:"account_id"` + AccountName string `json:"account_name"` + AccountSlug string `json:"account_slug"` + APIAuthKeyID string `json:"api_auth_key_id"` + BuildID string `json:"build_id"` + CompanionURL string `json:"companion_url"` + ExpectedTusUploads int `json:"expected_tus_uploads"` + FinishedTusUploads int `json:"finished_tus_uploads"` + IgnoredErrorCount int `json:"ignored_error_count"` + IgnoredErrors []AssemblyIgnoredError `json:"ignored_errors"` + Info map[string]interface{} `json:"info"` + Instance string `json:"instance"` + JobsQueueDuration float32 `json:"jobs_queue_duration"` + MergedParams string `json:"merged_params"` + NotifyError string `json:"notify_error"` + NotifyResponseCode Integer `json:"notify_response_code"` + NotifyResponseData string `json:"notify_response_data"` + NumInputFiles int `json:"num_input_files"` + QueueDuration float32 `json:"queue_duration"` + Region string `json:"region"` + RunningJobs []string `json:"running_jobs"` + StartedTusUploads int `json:"started_tus_uploads"` + Template string `json:"template"` + TemplateID string `json:"template_id"` + TemplateName string `json:"template_name"` + TransloaditClient string `json:"transloadit_client"` + TusUploads []TusUpload `json:"tus_uploads"` + TusURL string `json:"tus_url"` + UpdateStreamURL string `json:"update_stream_url"` + UploadMetaDataExtracted bool `json:"upload_meta_data_extracted"` + UppyServerURL string `json:"uppyserver_url"` + UsageTags string `json:"usage_tags"` + VirusName string `json:"virusname"` + Warnings []AssemblyWarning `json:"warnings"` + WebsocketURL string `json:"websocket_url"` + + // The following fields are only present when Error is set, giving + // additional diagnostic detail about the processing job that failed. + // Use Error for programmatic handling; these are for troubleshooting only. + Cmd interface{} `json:"cmd"` + ExitCode Integer `json:"exitCode"` + ExitSignal string `json:"exitSignal"` + File string `json:"file"` + Headers map[string]interface{} `json:"headers"` + IsPrivateAddress bool `json:"is_private_address"` + Name string `json:"name"` + NumRetries int `json:"numRetries"` + PlaywrightErrorCode string `json:"playwright_error_code"` + Reason interface{} `json:"reason"` + Retries int `json:"retries"` + Retryable bool `json:"retryable"` + ResponseCode Integer `json:"response_code"` + Stderr string `json:"stderr"` + Stdout string `json:"stdout"` + URL string `json:"url"` + URLHost string `json:"url_host"` +} + +// AssemblyIgnoredError describes a non-fatal error that occurred while +// processing an assembly but did not cause it to fail. +type AssemblyIgnoredError struct { + Error interface{} `json:"error"` + Message string `json:"message"` + Phase string `json:"phase"` + Step string `json:"step"` +} + +// AssemblyWarning describes a non-fatal warning raised while processing an +// assembly. +type AssemblyWarning struct { + Level string `json:"level"` + Msg string `json:"msg"` +} + +// TusUpload contains details about a single resumable (tus) upload belonging +// to an assembly. +type TusUpload struct { + Fieldname string `json:"fieldname"` + Filename string `json:"filename"` + Finished bool `json:"finished"` + Offset float64 `json:"offset"` + Size float64 `json:"size"` + UploadURL string `json:"upload_url"` + UserMeta map[string]interface{} `json:"user_meta"` } // FileInfo contains details about a file which was either uploaded or is the diff --git a/bearer_token.go b/bearer_token.go new file mode 100644 index 0000000..d178a12 --- /dev/null +++ b/bearer_token.go @@ -0,0 +1,59 @@ +package transloadit + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" +) + +// BearerTokenRequest contains options for exchanging a Client's Auth Key and +// Auth Secret for a scoped bearer token via Client.IssueBearerToken. +type BearerTokenRequest struct { + // Scope restricts the token to a space-separated list of scopes, e.g. + // "assemblies:read assemblies:write". If empty, the token inherits all + // scopes granted to the Auth Key. + Scope string + // Audience sets the optional token audience. If empty, the deployment's + // default audience is used. + Audience string +} + +// BearerToken contains a bearer token issued by the Transloadit API. Details +// about each value can be found at https://transloadit.com/docs/api/tokens/. +type BearerToken struct { + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` + Scope string `json:"scope"` + TokenType string `json:"token_type"` +} + +// IssueBearerToken exchanges the Client's Auth Key and Auth Secret for a +// scoped bearer token by calling POST /token. Unlike other requests made by +// this Client, the token endpoint is authenticated using HTTP Basic auth +// rather than the usual HMAC request signature. +func (client *Client) IssueBearerToken(ctx context.Context, tokenRequest BearerTokenRequest) (*BearerToken, error) { + form := url.Values{"grant_type": {"client_credentials"}} + if tokenRequest.Scope != "" { + form.Set("scope", tokenRequest.Scope) + } + if tokenRequest.Audience != "" { + form.Set("aud", tokenRequest.Audience) + } + + req, err := http.NewRequest("POST", client.config.Endpoint+"/token", strings.NewReader(form.Encode())) + if err != nil { + return nil, fmt.Errorf("issue bearer token: %s", err) + } + req = req.WithContext(ctx) + req.SetBasicAuth(client.config.AuthKey, client.config.AuthSecret) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + var token BearerToken + if err := client.doRequest(req, &token); err != nil { + return nil, err + } + + return &token, nil +} diff --git a/bearer_token_test.go b/bearer_token_test.go new file mode 100644 index 0000000..bf07f2c --- /dev/null +++ b/bearer_token_test.go @@ -0,0 +1,85 @@ +package transloadit + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" +) + +func TestIssueBearerToken_SendsBasicAuthAndFormBody(t *testing.T) { + client := NewClient(Config{ + AuthKey: "foo_key", + AuthSecret: "foo_secret", + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/token" { + t.Errorf("expected path /token, got %q", r.URL.Path) + } + + user, pass, ok := r.BasicAuth() + if !ok || user != "foo_key" || pass != "foo_secret" { + t.Errorf("expected basic auth foo_key/foo_secret, got %q/%q (ok=%v)", user, pass, ok) + } + + if err := r.ParseForm(); err != nil { + t.Fatal(err) + } + if got := r.Form.Get("grant_type"); got != "client_credentials" { + t.Errorf("expected grant_type=client_credentials, got %q", got) + } + if got := r.Form.Get("scope"); got != "assemblies:read" { + t.Errorf("expected scope=assemblies:read, got %q", got) + } + + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"opaque-token","expires_in":21600,"scope":"assemblies:read","token_type":"Bearer"}`)) + })) + defer server.Close() + + client.config.Endpoint = server.URL + + token, err := client.IssueBearerToken(context.Background(), BearerTokenRequest{Scope: "assemblies:read"}) + if err != nil { + t.Fatal(err) + } + + if token.AccessToken != "opaque-token" { + t.Errorf("expected access token %q, got %q", "opaque-token", token.AccessToken) + } + if token.ExpiresIn != 21600 { + t.Errorf("expected expires_in 21600, got %d", token.ExpiresIn) + } + if token.TokenType != "Bearer" { + t.Errorf("expected token_type Bearer, got %q", token.TokenType) + } +} + +func TestIssueBearerToken_ReturnsRequestError(t *testing.T) { + client := NewClient(Config{ + AuthKey: "foo_key", + AuthSecret: "foo_secret", + }) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":"GET_ACCOUNT_UNKNOWN_AUTH_KEY","message":"unknown auth key"}`)) + })) + defer server.Close() + + client.config.Endpoint = server.URL + + _, err := client.IssueBearerToken(context.Background(), BearerTokenRequest{}) + if err == nil { + t.Fatal("expected an error") + } + + reqErr, ok := err.(RequestError) + if !ok { + t.Fatalf("expected RequestError, got %T: %s", err, err) + } + if reqErr.Code != "GET_ACCOUNT_UNKNOWN_AUTH_KEY" { + t.Errorf("expected error code GET_ACCOUNT_UNKNOWN_AUTH_KEY, got %q", reqErr.Code) + } +} diff --git a/contract_test.go b/contract_test.go new file mode 100644 index 0000000..cfbb736 --- /dev/null +++ b/contract_test.go @@ -0,0 +1,212 @@ +package transloadit + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "sort" + "strings" + "testing" +) + +// TestOpenAPIContract guards against the Transloadit API drifting away from +// the request/response shapes BearerToken, BearerTokenRequest and +// AssemblyNotificationPayload were modeled on. It fetches the public OpenAPI +// spec published at https://api2.transloadit.com/openapi.json and fails if +// those schemas have changed, so a human notices before users do. +// See https://api2.transloadit.com/openapi.json +func TestOpenAPIContract(t *testing.T) { + spec := fetchOpenAPISpec(t) + + t.Run("token request schema", func(t *testing.T) { + schema := resolveSchema(t, spec, requestBodySchema(t, spec, "/token", "application/x-www-form-urlencoded")) + assertRequired(t, schema, "grant_type") + assertHasProperties(t, schema, "grant_type", "scope", "aud") + }) + + t.Run("token response schema", func(t *testing.T) { + schema := resolveSchema(t, spec, responseSchema(t, spec, "/token", "2XX", "application/json")) + assertRequired(t, schema, "access_token", "expires_in", "scope", "token_type") + }) + + t.Run("assembly notification payload schema", func(t *testing.T) { + webhook := mustIndex(t, mustIndex(t, spec, "webhooks"), "assemblyNotification") + post := mustIndex(t, webhook, "post") + payloadRef := mustIndex(t, mustIndex(t, mustIndex(t, post, "x-transloadit-webhook"), "payload"), "schema") + schema := resolveSchema(t, spec, payloadRef) + + branches, ok := schema["anyOf"].([]interface{}) + if !ok || len(branches) == 0 { + t.Fatalf("expected assemblyNotificationPayload to be an anyOf union, got %#v", schema) + } + + var gotRequired [][]string + for _, b := range branches { + branch := resolveSchema(t, spec, b) + gotRequired = append(gotRequired, stringSlice(branch["required"])) + // A sample of fields AssemblyInfo/AssemblyNotificationPayload rely + // on; not exhaustive, but enough to catch a reshaped payload. + assertHasProperties(t, branch, + "ok", "error", "assembly_id", "region", "instance", + "websocket_url", "update_stream_url", "tus_url", "account_id", + "notify_duration", "warnings", "results", "uploads") + } + + wantRequired := [][]string{{"ok"}, {"ok"}, {"error"}} + if !sameRequiredSets(gotRequired, wantRequired) { + t.Fatalf("assemblyNotificationPayload anyOf required fields changed: got %v, want %v", gotRequired, wantRequired) + } + }) +} + +func fetchOpenAPISpec(t *testing.T) map[string]interface{} { + t.Helper() + + res, err := http.Get("https://api2.transloadit.com/openapi.json") + if err != nil { + t.Fatalf("fetch openapi spec: %s", err) + } + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("read openapi spec: %s", err) + } + + var spec map[string]interface{} + if err := json.Unmarshal(body, &spec); err != nil { + t.Fatalf("decode openapi spec: %s", err) + } + + return spec +} + +func requestBodySchema(t *testing.T, spec map[string]interface{}, path, contentType string) interface{} { + t.Helper() + + op := mustIndex(t, mustIndex(t, mustIndex(t, spec, "paths"), path), "post") + body := mustIndex(t, op, "requestBody") + content := mustIndex(t, mustIndex(t, body, "content"), contentType) + return mustIndex(t, content, "schema") +} + +func responseSchema(t *testing.T, spec map[string]interface{}, path, status, contentType string) interface{} { + t.Helper() + + op := mustIndex(t, mustIndex(t, mustIndex(t, spec, "paths"), path), "post") + resp := mustIndex(t, mustIndex(t, op, "responses"), status) + content := mustIndex(t, mustIndex(t, resp, "content"), contentType) + return mustIndex(t, content, "schema") +} + +// mustIndex looks up key in a map[string]interface{} (or a schema wrapping +// one), failing the test with a readable path hint if it's missing. +func mustIndex(t *testing.T, node interface{}, key string) map[string]interface{} { + t.Helper() + + m, ok := node.(map[string]interface{}) + if !ok { + t.Fatalf("expected object while looking up %q, got %T", key, node) + } + + v, ok := m[key] + if !ok { + t.Fatalf("missing expected key %q in openapi spec (contract changed?)", key) + } + + child, ok := v.(map[string]interface{}) + if !ok { + t.Fatalf("expected %q to be an object, got %T", key, v) + } + + return child +} + +// resolveSchema follows a chain of "$ref"s pointing into +// #/components/schemas, if present, and returns the resolved schema object. +func resolveSchema(t *testing.T, spec map[string]interface{}, node interface{}) map[string]interface{} { + t.Helper() + + const prefix = "#/components/schemas/" + + for i := 0; i < 10; i++ { + schema, ok := node.(map[string]interface{}) + if !ok { + t.Fatalf("expected a schema object, got %T", node) + } + + ref, ok := schema["$ref"].(string) + if !ok { + return schema + } + + if !strings.HasPrefix(ref, prefix) { + t.Fatalf("unsupported $ref %q", ref) + } + + schemas := mustIndex(t, spec, "components")["schemas"] + node = mustIndex(t, schemas, strings.TrimPrefix(ref, prefix)) + } + + t.Fatalf("$ref chain too deep, possible cycle") + return nil +} + +func assertRequired(t *testing.T, schema map[string]interface{}, want ...string) { + t.Helper() + + got := stringSlice(schema["required"]) + sort.Strings(got) + sortedWant := append([]string{}, want...) + sort.Strings(sortedWant) + + if fmt.Sprint(got) != fmt.Sprint(sortedWant) { + t.Fatalf("required fields changed: got %v, want %v", got, sortedWant) + } +} + +func assertHasProperties(t *testing.T, schema map[string]interface{}, names ...string) { + t.Helper() + + properties, _ := schema["properties"].(map[string]interface{}) + for _, name := range names { + if _, ok := properties[name]; !ok { + t.Errorf("expected property %q to exist, it was removed or renamed", name) + } + } +} + +func stringSlice(v interface{}) []string { + list, ok := v.([]interface{}) + if !ok { + return nil + } + + out := make([]string, 0, len(list)) + for _, item := range list { + if s, ok := item.(string); ok { + out = append(out, s) + } + } + + return out +} + +func sameRequiredSets(got, want [][]string) bool { + if len(got) != len(want) { + return false + } + + for i := range got { + g := append([]string{}, got[i]...) + w := append([]string{}, want[i]...) + sort.Strings(g) + sort.Strings(w) + if fmt.Sprint(g) != fmt.Sprint(w) { + return false + } + } + + return true +} diff --git a/examples/webhook/main.go b/examples/webhook/main.go new file mode 100644 index 0000000..e5c7e28 --- /dev/null +++ b/examples/webhook/main.go @@ -0,0 +1,34 @@ +package main + +import ( + "fmt" + "log" + "net/http" + + "github.com/transloadit/go-sdk" +) + +// This example starts an HTTP server that receives Assembly Notifications. +// Configure your assembly's NotifyURL (see examples/image-resize) to point +// at this server's /webhook path to try it out. +func main() { + authSecret := "TRANSLOADIT_SECRET" + + http.HandleFunc("/webhook", func(w http.ResponseWriter, r *http.Request) { + notification, err := transloadit.ParseAssemblyNotification(r, authSecret) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + if notification.Error != "" { + fmt.Printf("assembly %s failed: %s\n", notification.AssemblyID, notification.Error) + } else { + fmt.Printf("assembly %s finished with status %s\n", notification.AssemblyID, notification.Ok) + } + + w.WriteHeader(http.StatusOK) + }) + + log.Fatal(http.ListenAndServe(":8080", nil)) +} diff --git a/webhook.go b/webhook.go new file mode 100644 index 0000000..882f4b3 --- /dev/null +++ b/webhook.go @@ -0,0 +1,54 @@ +package transloadit + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" +) + +// AssemblyNotificationPayload is the payload Transloadit submits to an +// assembly's notify_url once it finishes. Its shape matches AssemblyInfo, the +// result of GetAssembly. Use ParseAssemblyNotification to verify and decode +// an incoming notification request. +// See https://transloadit.com/docs/topics/assembly-instructions/#notifications +type AssemblyNotificationPayload struct { + AssemblyInfo +} + +// ParseAssemblyNotification verifies the signature of an incoming Assembly +// Notification request and decodes its payload. authSecret must be the +// Client's Config.AuthSecret. +// +// Transloadit sends notifications as a form-urlencoded POST request with the +// JSON payload in a "transloadit" field and a hex-encoded HMAC-SHA1 +// signature, computed over that field using authSecret, in a "signature" +// field. A successful HTTP response only acknowledges delivery; check +// AssemblyNotificationPayload.Ok and .Error for the assembly's outcome. +func ParseAssemblyNotification(r *http.Request, authSecret string) (*AssemblyNotificationPayload, error) { + if err := r.ParseForm(); err != nil { + return nil, fmt.Errorf("parse assembly notification: %s", err) + } + + payload := r.FormValue("transloadit") + signature := r.FormValue("signature") + if payload == "" || signature == "" { + return nil, fmt.Errorf("parse assembly notification: missing transloadit or signature field") + } + + hash := hmac.New(sha1.New, []byte(authSecret)) + hash.Write([]byte(payload)) + expected := hex.EncodeToString(hash.Sum(nil)) + if !hmac.Equal([]byte(signature), []byte(expected)) { + return nil, fmt.Errorf("parse assembly notification: signature mismatch") + } + + var result AssemblyNotificationPayload + if err := json.Unmarshal([]byte(payload), &result); err != nil { + return nil, fmt.Errorf("parse assembly notification: %s", err) + } + + return &result, nil +} diff --git a/webhook_test.go b/webhook_test.go new file mode 100644 index 0000000..074d331 --- /dev/null +++ b/webhook_test.go @@ -0,0 +1,73 @@ +package transloadit + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" +) + +const examplePayload = `{"ok":"ASSEMBLY_COMPLETED","assembly_id":"76fe5df1c93a0a530f3e583805cf98b4","assembly_url":"https://api2.transloadit.com/assemblies/76fe5df1c93a0a530f3e583805cf98b4","region":"us-east-1","results":{"resize":[{"id":"f1","name":"lol_cat.jpg","basename":"lol_cat","ext":"jpg","size":1234,"mime":"image/jpeg","type":"image","field":"image","url":"https://example.com/lol_cat.jpg","ssl_url":"https://example.com/lol_cat.jpg"}]}}` + +func signPayload(authSecret, payload string) string { + hash := hmac.New(sha1.New, []byte(authSecret)) + hash.Write([]byte(payload)) + return hex.EncodeToString(hash.Sum(nil)) +} + +func newNotificationRequest(payload, signature string) *http.Request { + form := url.Values{"transloadit": {payload}, "signature": {signature}} + req := httptest.NewRequest("POST", "/webhook", strings.NewReader(form.Encode())) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + return req +} + +func TestParseAssemblyNotification_VerifiesSignatureAndUnmarshals(t *testing.T) { + const authSecret = "foo_secret" + signature := signPayload(authSecret, examplePayload) + + req := newNotificationRequest(examplePayload, signature) + + notification, err := ParseAssemblyNotification(req, authSecret) + if err != nil { + t.Fatal(err) + } + + if notification.Ok != "ASSEMBLY_COMPLETED" { + t.Errorf("expected ok=ASSEMBLY_COMPLETED, got %q", notification.Ok) + } + if notification.AssemblyID != "76fe5df1c93a0a530f3e583805cf98b4" { + t.Errorf("expected assembly_id to be set, got %q", notification.AssemblyID) + } + if notification.Region != "us-east-1" { + t.Errorf("expected region=us-east-1, got %q", notification.Region) + } + if len(notification.Results["resize"]) != 1 || notification.Results["resize"][0].URL != "https://example.com/lol_cat.jpg" { + t.Errorf("expected one resize result with the expected URL, got %#v", notification.Results["resize"]) + } +} + +func TestParseAssemblyNotification_RejectsTamperedPayload(t *testing.T) { + const authSecret = "foo_secret" + signature := signPayload(authSecret, examplePayload) + + tampered := strings.Replace(examplePayload, "ASSEMBLY_COMPLETED", "ASSEMBLY_CANCELED", 1) + req := newNotificationRequest(tampered, signature) + + if _, err := ParseAssemblyNotification(req, authSecret); err == nil { + t.Fatal("expected signature verification to fail for a tampered payload") + } +} + +func TestParseAssemblyNotification_RejectsWrongSecret(t *testing.T) { + signature := signPayload("foo_secret", examplePayload) + req := newNotificationRequest(examplePayload, signature) + + if _, err := ParseAssemblyNotification(req, "wrong_secret"); err == nil { + t.Fatal("expected signature verification to fail for the wrong auth secret") + } +}