-
Notifications
You must be signed in to change notification settings - Fork 21
feat(metrics): add metrics plugin poc #145
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 2 commits
27cfc2f
7da4bac
ac133ee
e426e0c
6871534
1db7714
f51d80d
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| # metrics Plugin — Agent Context | ||
|
|
||
| Standalone HTTP API that queries DevLake's MySQL database and exposes pre-computed | ||
| metrics. Not part of the DevLake plugin framework — ships as an independent binary. | ||
|
|
||
| ## Build & Run | ||
|
|
||
| ```bash | ||
| # From backend/ | ||
| make build-metrics-api | ||
|
|
||
| # Run locally | ||
| export MYSQL_HOST=<MYSQL_HOST> | ||
| export MYSQL_PORT=<MYSQL_PORT> | ||
| export MYSQL_USER=<MYSQL_USER> | ||
| export MYSQL_PASS=<MYSQL_PASS> | ||
| export MYSQL_DB=<MYSQL_DB> | ||
| export METRICS_ADDR=:<METRICS_ADDR> | ||
| export METRICS_ALLOWED_ORIGIN=<METRICS_ALLOWED_ORIGIN> | ||
| ./bin/metrics-api | ||
|
|
||
| # Test | ||
| go test ./plugins/metrics/... -v | ||
| golangci-lint run ./plugins/metrics/... | ||
| ``` | ||
|
|
||
| ## Layout | ||
|
|
||
| ``` | ||
| cmd/ | ||
| main.go | ||
| api/ | ||
| server.go | ||
| middleware.go | ||
| params.go | ||
| routes/pr/ | ||
| register.go | ||
| handlers/ | ||
| utils.go | ||
| key_metrics.go | ||
| stages.go | ||
| cycle_time.go | ||
| productivity.go | ||
| flow.go | ||
| zscore.go | ||
| scatter.go | ||
| model/ | ||
| response.go | ||
| query/ | ||
| db.go | ||
| pr/ | ||
| rows.go | ||
| build.go | ||
| queries.go | ||
| testsupport/ | ||
| time.go | ||
| transform/ | ||
| botfilter/ | ||
| botfilter.go | ||
| pr/ | ||
| helpers.go | ||
| key_metrics.go | ||
| stages.go | ||
| cycle_time.go | ||
| flow.go | ||
| zscore.go | ||
| productivity.go | ||
| scatter.go | ||
| *_test.go | ||
| ``` | ||
|
|
||
| ## URL Scheme | ||
|
|
||
| All routes: `POST /api/metrics/<category>/<name>` | ||
|
|
||
| | Category | Example path | | ||
| |---|---| | ||
| | PR | `/api/metrics/pr/key-metrics` | | ||
|
|
||
|
|
||
| ## Request Body | ||
|
|
||
| All endpoints accept the same JSON body: | ||
|
|
||
| ```json | ||
| { | ||
| "owner": ["org"], | ||
| "name": ["repo"], | ||
| "from": 1700000000, | ||
| "to": 1702000000, | ||
| "blueprintid": "72", | ||
| "connectionid": "1", | ||
| "jiraproject": "PROJ", | ||
| "projects": ["project-name"], | ||
| "userwhitelist": [], | ||
| "teamrepos": [], | ||
| "projectname": "PRCT - Team" | ||
| } | ||
| ``` | ||
|
|
||
| ## Response Shape | ||
|
|
||
| All endpoints return FORMAT.md JSON — see `model/response.go` for Go types. | ||
|
|
||
| ## Conventions | ||
|
|
||
| - Transforms are pure functions — no I/O, no DB; takes rows → returns `model.MetricResponse` | ||
| - SQL lives in `query/pr/queries.go`; no SQL in handlers or transform files | ||
| - New SQL fragments use string concatenation (`+`) not `fmt.Sprintf` — see existing queries for the pattern | ||
| - Unit tests live next to source: `transform/pr/foo.go` → `transform/pr/foo_test.go` | ||
| - Use `testsupport.ParseTime()` for `*time.Time` fixtures in tests | ||
| - This binary is excluded from `build-plugins.sh` — build separately with `go build ./plugins/metrics/cmd/` | ||
| - Apache 2.0 license header required on all `.go` files | ||
|
|
||
|
|
||
| ## Dont's | ||
|
|
||
| - Don't import from other plugins (plugins must be independent) | ||
| - Don't skip the Apache 2.0 license header on new `.go` files |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| FROM golang:1.26-alpine AS builder | ||
|
|
||
| WORKDIR /build | ||
|
|
||
| # Copy go module files first for layer caching | ||
| COPY go.mod go.sum ./ | ||
| RUN go mod download | ||
|
|
||
| # Copy source | ||
| COPY . . | ||
|
|
||
| # Build the standalone binary | ||
| RUN CGO_ENABLED=0 GOOS=linux go build \ | ||
| -ldflags="-s -w" \ | ||
| -o /metrics-api \ | ||
| ./plugins/metrics/cmd/ | ||
|
|
||
| # ---- runtime image ---- | ||
| FROM gcr.io/distroless/static:nonroot | ||
|
|
||
| COPY --from=builder /metrics-api /metrics-api | ||
|
|
||
| EXPOSE 8181 | ||
| ENTRYPOINT ["/metrics-api"] | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| /* | ||
| Licensed to the Apache Software Foundation (ASF) under one or more | ||
| contributor license agreements. See the NOTICE file distributed with | ||
| this work for additional information regarding copyright ownership. | ||
| The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| (the "License"); you may not use this file except in compliance with | ||
| the License. You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "log" | ||
| "net/http" | ||
| "time" | ||
| ) | ||
|
|
||
| // corsMiddleware adds CORS headers and handles pre-flight OPTIONS requests. | ||
| func corsMiddleware(allowedOrigin string, next http.Handler) http.Handler { | ||
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| w.Header().Set("Access-Control-Allow-Origin", allowedOrigin) | ||
| w.Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS") | ||
| w.Header().Set("Access-Control-Allow-Headers", "Content-Type") | ||
| if r.Method == http.MethodOptions { | ||
| w.WriteHeader(http.StatusNoContent) | ||
| return | ||
| } | ||
| next.ServeHTTP(w, r) | ||
| }) | ||
| } | ||
|
|
||
| // loggingMiddleware logs method, path, status, and latency for every request. | ||
| func loggingMiddleware(next http.Handler) http.Handler { | ||
| return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { | ||
| start := time.Now() | ||
| rw := &responseWriter{ResponseWriter: w, status: http.StatusOK} | ||
| next.ServeHTTP(rw, r) | ||
| log.Printf("metrics-api: %s %s %d %s", r.Method, sanitizePath(r.URL.Path), rw.status, time.Since(start)) //nolint:gosec // G706: path is sanitized by sanitizePath (strips \n and \r) | ||
| }) | ||
| } | ||
|
|
||
| type responseWriter struct { | ||
| http.ResponseWriter | ||
| status int | ||
| } | ||
|
|
||
| func (rw *responseWriter) WriteHeader(status int) { | ||
| rw.status = status | ||
| rw.ResponseWriter.WriteHeader(status) | ||
| } | ||
|
|
||
| // sanitizePath strips newlines and carriage returns from a URL path to prevent | ||
| // log injection (gosec G706). | ||
| func sanitizePath(p string) string { | ||
| out := make([]byte, 0, len(p)) | ||
| for i := range len(p) { | ||
| if p[i] != '\n' && p[i] != '\r' { | ||
| out = append(out, p[i]) | ||
| } | ||
| } | ||
| return string(out) | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| /* | ||
| Licensed to the Apache Software Foundation (ASF) under one or more | ||
| contributor license agreements. See the NOTICE file distributed with | ||
| this work for additional information regarding copyright ownership. | ||
| The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| (the "License"); you may not use this file except in compliance with | ||
| the License. You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package api | ||
|
|
||
| import ( | ||
| "encoding/json" | ||
| "fmt" | ||
| "net/http" | ||
| ) | ||
|
|
||
| // QueryParams holds the normalized request body sent by the dashboard. | ||
| // All fields mirror what dashboard.js builds in buildDiagramRequest(). | ||
| type QueryParams struct { | ||
| // Owner is the list of GitHub/GitLab organisation names. | ||
| Owner []string `json:"owner"` | ||
| // Name is the list of repository names (parallel to Owner). | ||
| Name []string `json:"name"` | ||
| // From is the start of the time window as a Unix timestamp in seconds. | ||
| From int64 `json:"from"` | ||
| // To is the end of the time window as a Unix timestamp in seconds. | ||
| To int64 `json:"to"` | ||
| // BlueprintID is the DevLake blueprint ID for this team/product. | ||
| BlueprintID string `json:"blueprintid"` | ||
| // ConnectionID is the DevLake connection ID for the primary data source. | ||
| ConnectionID string `json:"connectionid"` | ||
| // JiraProject is the Jira project key (only for issue dashboards). | ||
| JiraProject string `json:"jiraproject"` | ||
| // Projects is used by issue dashboards in place of owner/name pairs. | ||
| Projects []string `json:"projects"` | ||
| // UserWhitelist restricts metrics to a specific set of contributors. | ||
| UserWhitelist []string `json:"userwhitelist"` | ||
| // TeamRepos is an explicit repo list used by some AR flows. | ||
| TeamRepos []string `json:"teamrepos"` | ||
| // ProjectName is the DevLake project_name scoping key (used by AI Review, AICS). | ||
| ProjectName string `json:"projectname"` | ||
| } | ||
|
|
||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] comment-accuracy Doc comment says reads and validates but function only decodes JSON — no validation is performed. |
||
| // DecodeQueryParams reads and validates the POST body into a QueryParams value. | ||
| // Returns an error suitable for returning as a 400 to the caller. | ||
| func DecodeQueryParams(r *http.Request) (QueryParams, error) { | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [medium] missing-input-validation DecodeQueryParams performs no input validation: no request body size limit (allows unbounded memory allocation), no check that From > 0, To > From, or BlueprintID is non-empty, and no cap on array lengths. Empty BlueprintID causes FIND_IN_SET to return 0 for every row. Unbounded arrays could generate very large SQL queries. Suggested fix: Wrap r.Body with http.MaxBytesReader. Validate From > 0, To > From, BlueprintID non-empty. Cap array lengths. |
||
| var p QueryParams | ||
| if err := json.NewDecoder(r.Body).Decode(&p); err != nil { | ||
| return p, fmt.Errorf("decoding request body: %w", err) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [low] info-disclosure JSON decode errors returned raw to client, revealing internal struct field names. Return generic error message instead. |
||
| } | ||
| return p, nil | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| /* | ||
| Licensed to the Apache Software Foundation (ASF) under one or more | ||
| contributor license agreements. See the NOTICE file distributed with | ||
| this work for additional information regarding copyright ownership. | ||
| The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| (the "License"); you may not use this file except in compliance with | ||
| the License. You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package pr | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "log" | ||
| "net/http" | ||
|
|
||
| "github.com/apache/incubator-devlake/plugins/metrics/api" | ||
| qpr "github.com/apache/incubator-devlake/plugins/metrics/query/pr" | ||
| tpr "github.com/apache/incubator-devlake/plugins/metrics/transform/pr" | ||
| ) | ||
|
|
||
| // CycleTime handles POST /api/metrics/pr/cycle-time. | ||
| func CycleTime(db *sql.DB) http.HandlerFunc { | ||
| return func(w http.ResponseWriter, r *http.Request) { | ||
| p, err := api.DecodeQueryParams(r) | ||
| if err != nil { | ||
| http.Error(w, err.Error(), http.StatusBadRequest) | ||
| return | ||
| } | ||
| prs, err := qpr.BasePRs(r.Context(), db, toParams(p)) | ||
| if err != nil { | ||
| log.Printf("pr/cycle-time: %v", err) | ||
| http.Error(w, "query error", http.StatusInternalServerError) | ||
| return | ||
| } | ||
| api.WriteJSON(w, tpr.BuildCycleTime(prs)) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| Licensed to the Apache Software Foundation (ASF) under one or more | ||
| contributor license agreements. See the NOTICE file distributed with | ||
| this work for additional information regarding copyright ownership. | ||
| The ASF licenses this file to You under the Apache License, Version 2.0 | ||
| (the "License"); you may not use this file except in compliance with | ||
| the License. You may obtain a copy of the License at | ||
|
|
||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
|
|
||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package pr | ||
|
|
||
| import ( | ||
| "database/sql" | ||
| "log" | ||
| "net/http" | ||
|
|
||
| "github.com/apache/incubator-devlake/plugins/metrics/api" | ||
| qpr "github.com/apache/incubator-devlake/plugins/metrics/query/pr" | ||
| tpr "github.com/apache/incubator-devlake/plugins/metrics/transform/pr" | ||
| ) | ||
|
|
||
| // Flow handles POST /api/metrics/pr/flow. | ||
| func Flow(db *sql.DB) http.HandlerFunc { | ||
| return func(w http.ResponseWriter, r *http.Request) { | ||
| p, err := api.DecodeQueryParams(r) | ||
| if err != nil { | ||
| http.Error(w, err.Error(), http.StatusBadRequest) | ||
| return | ||
| } | ||
| params := toParams(p) | ||
|
|
||
| prs, err := qpr.BasePRs(r.Context(), db, params) | ||
| if err != nil { | ||
| log.Printf("pr/flow: BasePRs: %v", err) | ||
| http.Error(w, "query error", http.StatusInternalServerError) | ||
| return | ||
| } | ||
| reviews, err := qpr.Reviews(r.Context(), db, params) | ||
| if err != nil { | ||
| log.Printf("pr/flow: Reviews: %v", err) | ||
| http.Error(w, "query error", http.StatusInternalServerError) | ||
| return | ||
| } | ||
| api.WriteJSON(w, tpr.BuildFlow(prs, reviews)) | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[low] logic-error
Uses FROM golang:1.26-alpine. Verify this tag exists as a published Docker image; if not, builds will fail.