diff --git a/.codespellrc b/.codespellrc new file mode 100644 index 00000000000..a10c0a3c634 --- /dev/null +++ b/.codespellrc @@ -0,0 +1,28 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[codespell] +# Ref: https://github.com/codespell-project/codespell#using-a-config-file +skip = .git,.gitignore,.gitattributes,*.svg,go.sum,*.lock,*.css,.codespellrc,.cache,.npm,.yarn,*/e2e/raw_tables/*,*/e2e/snapshot_tables/* +check-hidden = true +# Ignore camelCase and PascalCase identifiers (common in Go and TypeScript code) +ignore-regex = \b[a-z]+[A-Z]\w*\b|\b[A-Z][a-z]+[A-Z]\w*\b +# convertor,convertors - project's deliberate spelling for Go types and filenames (DataConvertor, etc.) +# crypted - variable name in AES encrypt/decrypt functions +# te - Tapd API field name (Te/te struct fields) +# thur - Thursday abbreviation in Grafana dashboard SQL column alias +ignore-words-list = convertor,convertors,crypted,te,thur diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f48cdbef2e9..5909efd372b 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM mcr.microsoft.com/devcontainers/go:1-1.23-bookworm +FROM mcr.microsoft.com/devcontainers/go:1-1.26-bookworm ENV PKG_CONFIG_PATH=$PKG_CONFIG_PATH:/usr/local/lib:/usr/local/lib/pkgconfig ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a9e43722772..94b84f20f66 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -15,7 +15,6 @@ * limitations under the License. * */ - { "name": "Go", "dockerComposeFile": "docker-compose.yml", @@ -36,6 +35,10 @@ 4000, 8080 ], + "containerEnv": { + "VITE_GRAFANA_URL": "http://grafana:3000", + "VITE_GRAFANA_CHANGE_ORIGIN": "false" + }, "postCreateCommand": "npm install commitizen -g", "customizations": { "vscode": { @@ -45,4 +48,4 @@ } }, "remoteUser": "root" -} +} \ No newline at end of file diff --git a/.github/actions/auto-cherry-pick/action.yml b/.github/actions/auto-cherry-pick/action.yml index 94eafe36f81..054ffbcf39d 100644 --- a/.github/actions/auto-cherry-pick/action.yml +++ b/.github/actions/auto-cherry-pick/action.yml @@ -19,7 +19,7 @@ name: "Auto Cherry Pick" description: "cherry pick commits from Pull Requests into Release branch" inputs: trigger_label_prefix: - description: "The trigger label prefic" + description: "The trigger label prefix" default: "needs-cherrypick-" required: false author_email: diff --git a/.github/workflows/NOTICE-year-check.yml b/.github/workflows/NOTICE-year-check.yml index 2fff55c63f1..07be9765f17 100644 --- a/.github/workflows/NOTICE-year-check.yml +++ b/.github/workflows/NOTICE-year-check.yml @@ -22,6 +22,6 @@ jobs: notice-year-check: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: check the NOTICE file run: grep $(date +"%Y") NOTICE diff --git a/.github/workflows/auto-cherry-pick.yml b/.github/workflows/auto-cherry-pick.yml index 39de99faa64..4668a0bc484 100644 --- a/.github/workflows/auto-cherry-pick.yml +++ b/.github/workflows/auto-cherry-pick.yml @@ -28,7 +28,7 @@ jobs: if: github.event.pull_request.merged == true runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 with: fetch-depth: 0 - name: Auto Cherry Pick diff --git a/.github/workflows/build-builder.yml b/.github/workflows/build-builder.yml index abdc25f2ca4..2c01e63c43e 100644 --- a/.github/workflows/build-builder.yml +++ b/.github/workflows/build-builder.yml @@ -25,7 +25,7 @@ jobs: name: Build lake-builder image runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 with: username: ${{ secrets.DOCKER_REGISTRY_USERNAME }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bde6a04ff51..6d7dc413728 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,18 +42,18 @@ jobs: sudo rm -rf /opt/hostedtoolcache/CodeQL docker system prune -af docker volume prune -f - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} - name: Build and push lake image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: ./backend push: true @@ -75,18 +75,18 @@ jobs: sudo rm -rf /opt/hostedtoolcache/CodeQL docker system prune -af docker volume prune -f - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} - name: Build and push lake image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: ./backend push: true @@ -112,25 +112,25 @@ jobs: sudo rm -rf /opt/hostedtoolcache/CodeQL docker system prune -af docker volume prune -f - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Get short sha id: get_short_sha run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} - - uses: actions/cache@v3 + - uses: actions/cache@v5 with: path: /tmp/devlake-build-cache-${{ matrix.platform }} key: buildx-devlake-build-cache-${{ github.run_id }}-${{ matrix.platform }} - name: Build and cache lake build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: ./backend push: false @@ -166,24 +166,24 @@ jobs: sudo rm -rf /opt/hostedtoolcache/CodeQL docker system prune -af docker volume prune -f - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Get short sha id: get_short_sha run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} - - uses: actions/cache@v3 + - uses: actions/cache@v5 with: path: /tmp/devlake-build-cache-amd64 key: buildx-devlake-build-cache-${{ github.run_id }}-amd64 - - uses: actions/cache@v3 + - uses: actions/cache@v5 with: path: /tmp/devlake-build-cache-arm64 key: buildx-devlake-build-cache-${{ github.run_id }}-arm64 @@ -199,7 +199,7 @@ jobs: echo "TAGS=${image_name}:${{ github.ref_name }}_${{ needs.get-timestamp.outputs.timestamp }}_${{ steps.get_short_sha.outputs.SHORT_SHA }}" >> $GITHUB_OUTPUT fi - name: Build and push lake image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: ./backend push: true @@ -214,7 +214,7 @@ jobs: TAG=${{ github.ref_name }} SHA=${{ steps.get_short_sha.outputs.SHORT_SHA }} - name: Clear cache - uses: actions/github-script@v6 + uses: actions/github-script@v9 if: always() with: script: | @@ -254,16 +254,16 @@ jobs: sudo rm -rf /opt/hostedtoolcache/CodeQL docker system prune -af docker volume prune -f - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Get short sha id: get_short_sha run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: username: ${{ env.DOCKERHUB_USERNAME }} password: ${{ env.DOCKERHUB_TOKEN }} @@ -279,7 +279,7 @@ jobs: echo "TAGS=${image_name}:${{ github.ref_name }}_${{ needs.get-timestamp.outputs.timestamp }}_${{ steps.get_short_sha.outputs.SHORT_SHA }}" >> $GITHUB_OUTPUT fi - name: Build and push ${{ matrix.build.name }} image - uses: docker/build-push-action@v3 + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: context: ${{ matrix.build.context }} push: true diff --git a/grafana/provisioning/datasources/datasource.yml b/.github/workflows/codespell.yml.action-blocked-by-asf similarity index 66% rename from grafana/provisioning/datasources/datasource.yml rename to .github/workflows/codespell.yml.action-blocked-by-asf index c2afa5aa0ca..be468623373 100644 --- a/grafana/provisioning/datasources/datasource.yml +++ b/.github/workflows/codespell.yml.action-blocked-by-asf @@ -14,16 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# Codespell configuration is within .codespellrc +--- +name: Codespell -# config file version -apiVersion: 1 +on: + push: + branches: [main] + pull_request: + branches: [main] -datasources: - - name: mysql - type: mysql - url: $MYSQL_URL - database: $MYSQL_DATABASE - user: $MYSQL_USER - secureJsonData: - password: $MYSQL_PASSWORD - editable: false +permissions: + contents: read + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Codespell + uses: codespell-project/actions-codespell@v2 diff --git a/.github/workflows/config-ui.yml b/.github/workflows/config-ui.yml index d4152860d65..fcf52a19e25 100644 --- a/.github/workflows/config-ui.yml +++ b/.github/workflows/config-ui.yml @@ -25,9 +25,9 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Node.js - uses: actions/setup-node@v3 + uses: actions/setup-node@v6 with: node-version: '18' cache: 'yarn' diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 28ec222084e..98d6deebd3e 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -29,30 +29,20 @@ jobs: runs-on: ubuntu-latest container: mericodev/lake-builder:latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Cache golangci-lint id: cache-golangci-lint - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: golangci-lint key: ${{ runner.os }}-backend-golangci-lint - - name: Install mockery - run: | - # The lake-builder container ships /go/bin/mockery at v2.20.0 and - # /go/bin precedes /usr/local/bin on PATH, so we must overwrite the - # binary in /go/bin (not just install to /usr/local/bin) for the - # Makefile's version check to see v2.53.5. - curl -sL https://github.com/vektra/mockery/releases/download/v2.53.5/mockery_2.53.5_Linux_x86_64.tar.gz \ - | tar -xz -C /go/bin mockery - chmod +x /go/bin/mockery - mockery --version - name: generate mock run: | go version cd backend make mock - name: golangci-lint - uses: golangci/golangci-lint-action@v6 + uses: golangci/golangci-lint-action@v9 with: - version: v1.53.3 + version: v2.12.2 working-directory: ./backend diff --git a/.github/workflows/grafana-dashboards-check.yml b/.github/workflows/grafana-dashboards-check.yml index a25bf4b1d67..e46ecf9fb4e 100644 --- a/.github/workflows/grafana-dashboards-check.yml +++ b/.github/workflows/grafana-dashboards-check.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Check grafana dashboards whether using mysql uid run: | if grep '"type": "mysql"' grafana/dashboards/*; then diff --git a/.github/workflows/migration-script-lint.yml b/.github/workflows/migration-script-lint.yml index 0429f03796d..bba0befdafb 100644 --- a/.github/workflows/migration-script-lint.yml +++ b/.github/workflows/migration-script-lint.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest container: mericodev/lake-builder:latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: migration script linting run: | go version diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 17fcb20b5f1..e4a1123913a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -27,7 +27,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v4 + - uses: actions/stale@v10 with: # Stale Issues days-before-issue-stale: 60 diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 4b078f8f6c3..a376351650a 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -48,7 +48,7 @@ jobs: container: mericodev/lake-builder:latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 - run: git config --global --add safe.directory $(pwd) - name: Build Python run: | @@ -57,7 +57,7 @@ jobs: make build-python - name: Cache test-e2e id: cache-test-e2e - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: | ~/.cache/go-build @@ -88,7 +88,7 @@ jobs: container: mericodev/lake-builder:latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 - run: git config --global --add safe.directory $(pwd) - name: Build Python run: | @@ -97,7 +97,7 @@ jobs: make build-python - name: Cache test-e2e-pg id: cache-test-e2e-pg - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: | ~/.cache/go-build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f55c98d623..967d5ab7dcb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,7 @@ jobs: container: mericodev/lake-builder:latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v6 - name: Set git config run: git config --global --add safe.directory $(pwd) - name: Build Python @@ -54,7 +54,7 @@ jobs: make build-python - name: Cache unit-test id: cache-unit-test - uses: actions/cache@v3 + uses: actions/cache@v5 with: path: | ~/.cache/go-build @@ -62,16 +62,6 @@ jobs: key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} restore-keys: | ${{ runner.os }}-go- - - name: Install mockery - run: | - # The lake-builder container ships /go/bin/mockery at v2.20.0 and - # /go/bin precedes /usr/local/bin on PATH, so we must overwrite the - # binary in /go/bin (not just install to /usr/local/bin) for the - # Makefile's version check to see v2.53.5. - curl -sL https://github.com/vektra/mockery/releases/download/v2.53.5/mockery_2.53.5_Linux_x86_64.tar.gz \ - | tar -xz -C /go/bin mockery - chmod +x /go/bin/mockery - mockery --version - name: Unit tests run: | go version diff --git a/.github/workflows/yaml-lint.yml b/.github/workflows/yaml-lint.yml index 6c9dde504df..ce2555f58ef 100644 --- a/.github/workflows/yaml-lint.yml +++ b/.github/workflows/yaml-lint.yml @@ -26,7 +26,7 @@ jobs: name: lint for yamls runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - uses: github/super-linter/slim@v4 env: FILTER_REGEX_INCLUDE: workspace/(docker-compose.yml|deployment/k8s/k8s-deploy.yaml) diff --git a/.gitignore b/.gitignore index be94081169f..91167b40250 100644 --- a/.gitignore +++ b/.gitignore @@ -150,6 +150,9 @@ bin libgit2 .air.toml +# Playwright CLI snapshots/logs (local dev) +.playwright-cli/ + # auto generated code backend/mocks/ backend/server/api/docs/swagger.json diff --git a/Makefile b/Makefile index 5e18c277968..8639ce2dc6f 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ build-config-ui-image: cd config-ui; docker build -t $(IMAGE_REPO)/devlake-config-ui:$(TAG) --file ./Dockerfile . build-grafana-image: - cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./backend/Dockerfile . + cd grafana; docker build -t $(IMAGE_REPO)/devlake-dashboard:$(TAG) --file ./Dockerfile . build-images: build-server-image build-config-ui-image build-grafana-image diff --git a/README.md b/README.md index c31c0ffd3db..8dc8de01696 100644 --- a/README.md +++ b/README.md @@ -19,13 +19,13 @@ limitations under the License.
-# Apache DevLake(Incubating) +# Apache DevLake [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat&logo=github&color=2370ff&labelColor=454545)](http://makeapullrequest.com) [![Dockerhub pulls](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fhub.docker.com%2Fv2%2Frepositories%2Fapache%2Fdevlake&query=%24.pull_count&label=Dockerhub%20pulls)](https://hub.docker.com/r/apache/devlake) -[![unit-test](https://github.com/apache/incubator-devlake/actions/workflows/test.yml/badge.svg)](https://github.com/apache/incubator-devlake/actions/workflows/test.yml) -[![Go Report Card](https://goreportcard.com/badge/github.com/apache/incubator-devlake)](https://goreportcard.com/report/github.com/apache/incubator-devlake) -[![Slack](https://img.shields.io/badge/slack-join_chat-success.svg?logo=slack)](https://join.slack.com/t/devlake-io/shared_invite/zt-18uayb6ut-cHOjiYcBwERQ8VVPZ9cQQw) +[![unit-test](https://github.com/apache/devlake/actions/workflows/test.yml/badge.svg)](https://github.com/apache/devlake/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/apache/devlake)](https://goreportcard.com/report/github.com/apache/devlake) +[![Slack](https://img.shields.io/badge/slack-join_chat-success.svg?logo=slack)](https://join.slack.com/t/devlake-io/shared_invite/zt-1lkgbdmys-AU2azidzO1u~mtjlg9my7A) [![Twitter](https://badgen.net/badge/icon/twitter?icon=twitter&label)](https://twitter.com/ApacheDevLake) @@ -66,6 +66,7 @@ You can set up Apache DevLake by following our step-by-step instructions for eit - [Install via Docker Compose](https://devlake.apache.org/docs/GettingStarted/DockerComposeSetup) - [Install via Helm](https://devlake.apache.org/docs/GettingStarted/HelmSetup) +- [Install via gh-devlake CLI](https://github.com/DevExpGBB/gh-devlake) — A GitHub CLI extension that deploys, configures, and monitors DevLake entirely from the terminal. Supports local Docker and Azure deployments. ## 🤓 Usage @@ -95,10 +96,10 @@ If the pre-built dashboards are limited for your use cases, you can always custo Please read the [contribution guidelines](https://devlake.apache.org/community) before you make contribution. The following docs list the resources you might need to know after you decided to make contribution. - [Create an Issue](https://devlake.apache.org/community/MakingContributions/fix-or-create-issues): Report a bug or feature request to Apache DevLake -- [Submit a PR](https://devlake.apache.org/community/MakingContributions/development-workflow): Start with [good first issues](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) or [issues with no assignees](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue+is%3Aopen+no%3Aassignee) +- [Submit a PR](https://devlake.apache.org/community/MakingContributions/development-workflow): Start with [good first issues](https://github.com/apache/devlake/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) or [issues with no assignees](https://github.com/apache/devlake/issues?q=is%3Aissue+is%3Aopen+no%3Aassignee) - [Join Mailing list](https://devlake.apache.org/community/subscribe): Initiate or participate in project discussions on the mailing list - [Write a Blog](https://devlake.apache.org/community/MakingContributions/BlogSubmission): Write a blog to share your use cases about Apache DevLake -- [Develop a Plugin](./backend/DevelopmentManual): Integrate Apache DevLake with more data sources as [requested by the community](https://github.com/apache/incubator-devlake/issues?q=is%3Aissue+is%3Aopen+label%3Aadd-a-plugin+) +- [Develop a Plugin](./backend/DevelopmentManual): Integrate Apache DevLake with more data sources as [requested by the community](https://github.com/apache/devlake/issues?q=is%3Aissue+is%3Aopen+label%3Aadd-a-plugin+) ### 👩🏾‍💻 Contributing Code @@ -113,7 +114,7 @@ If you plan to contribute code to Apache DevLake, we have instructions on how to One of the best ways to get started contributing is by improving DevLake's documentation. - Apache DevLake's documentation is hosted at [devlake.apache.org](https://devlake.apache.org/) -- **We have a separate GitHub repository for Apache DevLake's documentation:** [github.com/apache/incubator-devlake-website](https://github.com/apache/incubator-devlake-website) +- **We have a separate GitHub repository for Apache DevLake's documentation:** [github.com/apache/devlake-website](https://github.com/apache/devlake-website) ## ⌚ Roadmap @@ -121,7 +122,7 @@ One of the best ways to get started contributing is by improving DevLake's docum ## 💙 Community -Message us on Slack +Message us on Slack ## 📄 License diff --git a/backend/.dockerignore b/backend/.dockerignore new file mode 100644 index 00000000000..36ee77aae07 --- /dev/null +++ b/backend/.dockerignore @@ -0,0 +1,24 @@ +# Build artifacts: regenerated inside the image (GOBIN=/app/bin + make build-plugin/build-server). +# Shipping the host's pre-built bin/ (multiple GB of plugin .so files) bloats the +# build context and the builder layer, which can exhaust the Docker VM and crash BuildKit. +bin/ + +# Generated mocks (regenerated via make mock inside the image) +mocks/ + +# Runtime logs +logs/ + +# Python build/venv artifacts (regenerated; Dockerfile.server reinstalls from +# requirements.txt / pyproject and runs python/build.sh). Source under python/ is kept. +**/.venv/ +**/__pycache__/ +**/*.pyc +python/.devlake-python-build-root/ + +# Test / coverage output +*.out +coverage.txt + +# OS / editor junk +.DS_Store diff --git a/backend/.golangci.yaml b/backend/.golangci.yaml index c0b2bad2ca5..fa4042b6bbb 100644 --- a/backend/.golangci.yaml +++ b/backend/.golangci.yaml @@ -1,3 +1,4 @@ +# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,322 +15,241 @@ # limitations under the License. # -# https://golangci-lint.run/usage/linters/ -# if you complain about it, you can communicate with us after reading https://github.com/golang/go/wiki/CodeReviewComments. +version: "2" +run: + concurrency: 4 + issues-exit-code: 2 + tests: false linters: - disable-all: true + default: none enable: - # - deadcode - errcheck - - gosimple + - goheader - govet + - importas - ineffassign + - makezero + - revive - staticcheck - # - structcheck - - typecheck - unused - # - varcheck - # - exhaustive - # - funlen - # - goconst - - gofmt - - goheader - # - goimports - # - gomnd - # - gosec - - importas - # - lll - # - misspell - # - nestif - # - prealloc - # - tagliatelle - - revive - # - dupl - - makezero - - stylecheck - -linters-settings: - goheader: - template-path: .golangci-goheader.template - - stylecheck: - # https://staticcheck.io/docs/options#checks - # Default: ["*"] - checks: [ "all", "-ST1000", "-ST1003", "-ST1005" ] - - revive: - # Maximum number of open files at the same time. - # See https://github.com/mgechev/revive#command-line-flags - # Defaults to unlimited. - max-open-files: 2048 - # Sets the default severity. - # See https://github.com/mgechev/revive#configuration - # Default: warning - severity: error - + settings: + goheader: + template-path: .golangci-goheader.template + govet: + disable: + - inline + revive: + max-open-files: 2048 + severity: error + rules: + - name: atomic + severity: warning + disabled: false + - name: bare-return + severity: warning + disabled: true + - name: blank-imports + severity: warning + disabled: false + - name: bool-literal-in-expr + severity: warning + disabled: false + - name: call-to-gc + severity: warning + disabled: false + - name: confusing-naming + severity: warning + disabled: false + - name: confusing-results + severity: warning + disabled: false + - name: constant-logical-expr + severity: warning + disabled: false + - name: context-as-argument + severity: warning + disabled: false + - name: context-keys-type + severity: warning + disabled: false + - name: deep-exit + severity: warning + disabled: false + - name: defer + arguments: + - - call-chain + severity: warning + disabled: false + - name: dot-imports + severity: warning + disabled: false + - name: duplicated-imports + severity: warning + disabled: false + - name: empty-block + severity: warning + disabled: false + - name: empty-lines + severity: warning + disabled: false + - name: error-naming + severity: warning + disabled: false + - name: error-return + severity: warning + disabled: false + - name: error-strings + severity: warning + disabled: false + - name: errorf + severity: warning + disabled: false + - name: exported + arguments: + - disableStutteringCheck + severity: warning + disabled: true + - name: get-return + severity: warning + disabled: false + - name: identical-branches + severity: warning + disabled: false + - name: import-shadowing + severity: warning + disabled: true + - name: modifies-parameter + severity: warning + disabled: false + - name: modifies-value-receiver + severity: warning + disabled: false + - name: optimize-operands-order + severity: warning + disabled: false + - name: package-comments + severity: warning + disabled: true + - name: range + severity: warning + disabled: false + - name: range-val-in-closure + severity: warning + disabled: false + - name: range-val-address + severity: warning + disabled: false + - name: receiver-naming + severity: warning + disabled: false + - name: redefines-builtin-id + severity: warning + disabled: false + - name: string-of-int + severity: warning + disabled: false + - name: string-format + arguments: + - - core.WriteError[1].Message + - /^([^A-Z]|$)/ + - must not start with a capital letter + - - fmt.Errorf[0] + - /(^|[^\.!?])$/ + - must not end in punctuation + - - panic + - /^[^\n]*$/ + - must not contain line breaks + severity: warning + disabled: false + - name: struct-tag + severity: warning + disabled: false + - name: superfluous-else + severity: warning + disabled: false + - name: time-equal + severity: warning + disabled: false + - name: time-naming + severity: warning + disabled: false + - name: var-naming + arguments: + - [] + - [] + severity: warning + disabled: true + - name: var-declaration + severity: warning + disabled: false + - name: unconditional-recursion + severity: warning + disabled: false + - name: unexported-return + severity: warning + disabled: false + - name: unhandled-error + arguments: + - fmt.Printf + - fmt.Println + - fmt.Fprintln + - res.Body.Close + - body.Close + severity: warning + disabled: false + - name: unnecessary-stmt + severity: warning + disabled: false + - name: unreachable-code + severity: warning + disabled: false + - name: useless-break + severity: warning + disabled: false + - name: waitgroup-by-value + severity: warning + disabled: false + staticcheck: + checks: + - all + - -ST1000 + - -ST1003 + - -ST1005 + - -ST1020 + - -ST1021 + - -ST1022 + - -QF1001 + - -QF1003 + - -QF1004 + - -QF1006 + - -QF1007 + - -QF1008 + - -QF1009 + - -QF1012 + exclusions: + generated: lax + presets: + - common-false-positives + - legacy + - std-error-handling rules: - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#atomic - - name: atomic - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bare-return - - name: bare-return - severity: warning - disabled: true - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#blank-imports - - name: blank-imports - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#bool-literal-in-expr - - name: bool-literal-in-expr - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#call-to-gc - - name: call-to-gc - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cognitive-complexity - # - name: cognitive-complexity - # severity: warning - # disabled: false - # arguments: [20] - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-naming - - name: confusing-naming - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#confusing-results - - name: confusing-results - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#constant-logical-expr - - name: constant-logical-expr - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-as-argument - - name: context-as-argument - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#context-keys-type - - name: context-keys-type - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#cyclomatic - # - name: cyclomatic - # severity: warning - # disabled: false - # arguments: [20] - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#deep-exit - - name: deep-exit - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#defer - - name: defer - severity: warning - disabled: false - arguments: - - [ "call-chain" ] - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#dot-imports - - name: dot-imports - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#duplicated-imports - - name: duplicated-imports - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#early-return - # - name: early-return - # severity: warning - # disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-block - - name: empty-block - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#empty-lines - - name: empty-lines - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-naming - - name: error-naming - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-return - - name: error-return - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#error-strings - - name: error-strings - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#errorf - - name: errorf - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#exported - - name: exported - severity: warning - disabled: true - arguments: - - "disableStutteringCheck" - # - "checkPrivateReceivers" - # - "sayRepetitiveInsteadOfStutters" - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#get-return - - name: get-return - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#identical-branches - - name: identical-branches - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#indent-error-flow - # - name: indent-error-flow - # severity: warning - # disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#import-shadowing - - name: import-shadowing - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-parameter - - name: modifies-parameter - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#modifies-value-receiver - - name: modifies-value-receiver - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#optimize-operands-order - - name: optimize-operands-order - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#package-comments - - name: package-comments - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range - - name: range - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-in-closure - - name: range-val-in-closure - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#range-val-address - - name: range-val-address - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#receiver-naming - - name: receiver-naming - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#redefines-builtin-id - - name: redefines-builtin-id - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-of-int - - name: string-of-int - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#string-format - - name: string-format - severity: warning - disabled: false - arguments: - - - 'core.WriteError[1].Message' - - '/^([^A-Z]|$)/' - - must not start with a capital letter - - - 'fmt.Errorf[0]' - - '/(^|[^\.!?])$/' - - must not end in punctuation - - - panic - - '/^[^\n]*$/' - - must not contain line breaks - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#struct-tag - - name: struct-tag - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#superfluous-else - - name: superfluous-else - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-equal - - name: time-equal - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#time-naming - - name: time-naming - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-naming - - name: var-naming - severity: warning - disabled: true - arguments: - - [ ] # AllowList - - [ ] # DenyList - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#var-declaration - - name: var-declaration - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unconditional-recursion - - name: unconditional-recursion - severity: warning - disabled: false - # # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-naming - # - name: unexported-naming - # severity: warning - # disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unexported-return - - name: unexported-return - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unhandled-error - - name: unhandled-error - severity: warning - disabled: false - arguments: - - "fmt.Printf" - - "fmt.Println" - - "fmt.Fprintln" - - "res.Body.Close" - - "body.Close" - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unnecessary-stmt - - name: unnecessary-stmt - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unreachable-code - - name: unreachable-code - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#useless-break - - name: useless-break - severity: warning - disabled: false - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#waitgroup-by-value - - name: waitgroup-by-value - severity: warning - disabled: false - -issues: - include: - - EXC0012 # EXC0012 revive: Annoying issue about not having a comment. The rare codebase has such comments - - EXC0014 # EXC0014 revive: Annoying issue about not having a comment. The rare codebase has such comments - exclude-rules: - - path: models/|api/|migration/|errors/|logger/ - linters: - - revive - - path: plugins/([^hc].*|.[^eo].*|..[^lr].*|...[^pe].*|....[^e].*|.....[^r].*) - linters: - - revive - -# Options for analysis running. -run: - # The default concurrency value is the number of available CPU. - concurrency: 4 - # Timeout for analysis, e.g. 30s, 5m. - # Default: 1m - timeout: 5m - # Exit code when at least one issue was found. - # Default: 1 - issues-exit-code: 2 - # Include test files or not. - # Default: true - tests: false + - linters: + - revive + path: models/|api/|migration/|errors/|logger/ + - linters: + - revive + path: plugins/([^hc].*|.[^eo].*|..[^lr].*|...[^pe].*|....[^e].*|.....[^r].*) + paths: + - third_party$ + - builtin$ + - examples$ + - mocks$ +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/backend/Dockerfile b/backend/Dockerfile index ef9236970aa..1d839f72e8c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -31,11 +31,7 @@ FROM --platform=linux/arm64 debian:bookworm as debian-arm64 RUN apt-get update RUN apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev -FROM --platform=$BUILDPLATFORM golang:1.23-bookworm as golang-tools -RUN go install github.com/vektra/mockery/v2@v2.53.5 -RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1 - -FROM --platform=$BUILDPLATFORM golang:1.20.5-bookworm as builder +FROM --platform=$BUILDPLATFORM golang:1.26-bookworm as builder # docker build --build-arg GOPROXY=https://goproxy.io,direct -t mericodev/lake . ARG GOPROXY= @@ -53,8 +49,8 @@ RUN if [ "$(arch)" != "x86_64" ] ; then \ apt-get install -y gcc-x86-64-linux-gnu binutils-x86-64-linux-gnu ; \ fi -COPY --from=golang-tools /go/bin/mockery /usr/local/bin/mockery -COPY --from=golang-tools /go/bin/swag /usr/local/bin/swag +RUN go install github.com/vektra/mockery/v2@v2.53.6 +RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 COPY --from=debian-amd64 /usr/include /rootfs-amd64/usr/include COPY --from=debian-amd64 /usr/lib/x86_64-linux-gnu /rootfs-amd64/usr/lib/x86_64-linux-gnu diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index d4dbb0c0763..2fadbcec90e 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -23,7 +23,7 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM golang:1.20.5-bookworm as builder +FROM golang:1.26-bookworm as builder ARG GOPROXY= ARG HTTP_PROXY= @@ -47,8 +47,8 @@ RUN mkdir -p /tmp/build && cd /tmp/build && \ make -j$(nproc) install && \ ldconfig -RUN go install github.com/vektra/mockery/v2@v2.20.0 -RUN go install github.com/swaggo/swag/cmd/swag@v1.16.1 +RUN go install github.com/vektra/mockery/v2@v2.53.6 +RUN go install github.com/swaggo/swag/cmd/swag@v1.16.6 WORKDIR /app COPY . /app diff --git a/backend/Makefile b/backend/Makefile index d7ac5bce3b2..2b4a1885712 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -28,9 +28,9 @@ MOCKERY_PINNED_VERSION ?= v2.53.5 all: build go-dep: - go install github.com/vektra/mockery/v2@$(MOCKERY_PINNED_VERSION) - go install github.com/swaggo/swag/cmd/swag@v1.16.1 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.63.4 + go install github.com/vektra/mockery/v2@v2.53.6 + go install github.com/swaggo/swag/cmd/swag@v1.16.6 + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 go-dev-tools: # go install github.com/atombender/go-jsonschema/cmd/gojsonschema@latest @@ -73,16 +73,6 @@ debug: build-plugin-debug dlv debug server/main.go mock: - @if ! command -v mockery >/dev/null 2>&1; then \ - echo "mockery is required. Install with: go install github.com/vektra/mockery/v2@$(MOCKERY_PINNED_VERSION)"; \ - exit 1; \ - fi - @MOCKERY_VERSION=$$(mockery --version 2>/dev/null | grep -Eo 'v[0-9]+\.[0-9]+\.[0-9]+' | head -n1); \ - if [ "$$MOCKERY_VERSION" != "$(MOCKERY_PINNED_VERSION)" ]; then \ - echo "mockery version $$MOCKERY_VERSION detected, expected $(MOCKERY_PINNED_VERSION)."; \ - echo "Install with: go install github.com/vektra/mockery/v2@$(MOCKERY_PINNED_VERSION)"; \ - exit 1; \ - fi rm -rf mocks mockery --recursive --keeptree --dir=./core --output=./mocks/core --unroll-variadic=false --name='.*' mockery --recursive --keeptree --dir=./helpers --output=./mocks/helpers --unroll-variadic=false --name='.*' diff --git a/backend/core/migration/linter/main.go b/backend/core/migration/linter/main.go index bf9190b72e1..11aae870ce9 100644 --- a/backend/core/migration/linter/main.go +++ b/backend/core/migration/linter/main.go @@ -51,7 +51,7 @@ func firstLineFromFile(path string) string { for scanner.Scan() { return scanner.Text() } - panic(fmt.Errorf("empty file: " + path)) + panic(fmt.Errorf("empty file: %s", path)) } const ( diff --git a/backend/core/models/common/iso8601time_test.go b/backend/core/models/common/iso8601time_test.go index 4307345da7b..84a33e8932a 100644 --- a/backend/core/models/common/iso8601time_test.go +++ b/backend/core/models/common/iso8601time_test.go @@ -167,7 +167,7 @@ func TestConvertStringToTime(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { output, err := ConvertStringToTime(tc.input) - if !reflect.DeepEqual(tc.output, output) { + if !tc.output.Equal(output) { t.Errorf("Expected output to be %v, but got %v", tc.output, output) } assert.Equal(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.err), "Expected error to be %v, but got %v", tc.err, err) @@ -322,7 +322,7 @@ func TestConvertStringToTimeInLoc(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { output, err := ConvertStringToTimeInLoc(tc.input, tc.loc) - if !reflect.DeepEqual(tc.output, output) { + if !tc.output.Equal(output) { t.Errorf("Expected output to be %v, but got %v", tc.output, output) } assert.Equal(t, fmt.Sprintf("%v", err), fmt.Sprintf("%v", tc.err), "Expected error to be %v, but got %v", tc.err, err) diff --git a/backend/core/models/domainlayer/README.md b/backend/core/models/domainlayer/README.md index e6b85d4bb9e..18f082db207 100644 --- a/backend/core/models/domainlayer/README.md +++ b/backend/core/models/domainlayer/README.md @@ -57,7 +57,7 @@ The following rules make sure Domain Layer Entities serve its purpose - Read data from platform specific table, convert and store record into one(or multiple) domain table(s) - Generate its own `Id` accordingly -- Generate foreign key accordlingly +- Generate foreign key accordingly - Fields conversion Sample code: diff --git a/backend/core/models/domainlayer/codequality/cq_file_metrics.go b/backend/core/models/domainlayer/codequality/cq_file_metrics.go index abc03b85f6f..dc069230e20 100644 --- a/backend/core/models/domainlayer/codequality/cq_file_metrics.go +++ b/backend/core/models/domainlayer/codequality/cq_file_metrics.go @@ -23,7 +23,7 @@ import ( type CqFileMetrics struct { domainlayer.DomainEntity - ProjectKey string `gorm:"index;type:varchar(255)"` //domain project key + ProjectKey string `gorm:"index;type:varchar(500)"` //domain project key FileName string `gorm:"type:varchar(2000)"` FilePath string FileLanguage string `gorm:"type:varchar(20)"` diff --git a/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go b/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go index 294bc5d3cb2..c522fc5cce0 100644 --- a/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go +++ b/backend/core/models/domainlayer/codequality/cq_issue_code_blocks.go @@ -22,7 +22,7 @@ import "github.com/apache/incubator-devlake/core/models/domainlayer" type CqIssueCodeBlock struct { domainlayer.DomainEntity IssueKey string `json:"key" gorm:"index"` - Component string `gorm:"index"` + Component string `gorm:"type:text"` StartLine int EndLine int StartOffset int diff --git a/backend/core/models/domainlayer/codequality/cq_issues.go b/backend/core/models/domainlayer/codequality/cq_issues.go index 1e9ea7134a7..4441eb1d7c0 100644 --- a/backend/core/models/domainlayer/codequality/cq_issues.go +++ b/backend/core/models/domainlayer/codequality/cq_issues.go @@ -27,7 +27,7 @@ type CqIssue struct { Rule string `gorm:"type:varchar(255)"` Severity string `gorm:"type:varchar(100)"` Component string - ProjectKey string `gorm:"index;type:varchar(100)"` //domain project key + ProjectKey string `gorm:"index;type:varchar(500)"` //domain project key Line int Status string `gorm:"type:varchar(20)"` Message string diff --git a/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go b/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go new file mode 100644 index 00000000000..90f64ce660c --- /dev/null +++ b/backend/core/models/domainlayer/codequality/cq_project_metrics_history.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package codequality + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/domainlayer" +) + +type CqProjectMetricsHistory struct { + domainlayer.DomainEntity + ProjectKey string `gorm:"index;type:varchar(500)"` + AnalysisDate time.Time `gorm:"index"` + Coverage *float64 + Ncloc *int + Bugs *int + ReliabilityRating string `gorm:"type:varchar(5)"` + CodeSmells *int + SqaleRating string `gorm:"type:varchar(5)"` + Complexity *int + CognitiveComplexity *int + Vulnerabilities *int + SecurityRating string `gorm:"type:varchar(5)"` + SecurityHotspots *int + DuplicatedLinesDensity *float64 +} + +func (CqProjectMetricsHistory) TableName() string { + return "cq_project_metrics_history" +} diff --git a/backend/core/models/domainlayer/devops/cicd_deployment.go b/backend/core/models/domainlayer/devops/cicd_deployment.go index b3071077253..e1f9049ded6 100644 --- a/backend/core/models/domainlayer/devops/cicd_deployment.go +++ b/backend/core/models/domainlayer/devops/cicd_deployment.go @@ -24,7 +24,7 @@ import ( type CICDDeployment struct { domainlayer.DomainEntity CicdScopeId string `gorm:"index;type:varchar(255)"` - Name string `gorm:"type:varchar(255)"` + Name string `gorm:"type:text"` DisplayTitle string Url string Result string `gorm:"type:varchar(100)"` diff --git a/backend/core/models/domainlayer/domaininfo/domaininfo.go b/backend/core/models/domainlayer/domaininfo/domaininfo.go index b2cc5431fb2..b88289e8d8f 100644 --- a/backend/core/models/domainlayer/domaininfo/domaininfo.go +++ b/backend/core/models/domainlayer/domaininfo/domaininfo.go @@ -56,6 +56,7 @@ func GetDomainTablesInfo() []dal.Tabler { &codequality.CqIssue{}, &codequality.CqIssueImpact{}, &codequality.CqProject{}, + &codequality.CqProjectMetricsHistory{}, // crossdomain &crossdomain.Account{}, &crossdomain.BoardRepo{}, diff --git a/backend/core/models/domainlayer/qa/qa_test_case_execution.go b/backend/core/models/domainlayer/qa/qa_test_case_execution.go index cfab89b6d05..aaf237b6fce 100644 --- a/backend/core/models/domainlayer/qa/qa_test_case_execution.go +++ b/backend/core/models/domainlayer/qa/qa_test_case_execution.go @@ -33,6 +33,7 @@ type QaTestCaseExecution struct { FinishTime time.Time `gorm:"comment:Test finish time"` CreatorId string `gorm:"type:varchar(255);comment:Executor ID"` Status string `gorm:"type:varchar(255);comment:Test execution status | PENDING | IN_PROGRESS | SUCCESS | FAILED"` // enum, using string + IsInvalid bool } func (QaTestCaseExecution) TableName() string { diff --git a/backend/core/models/locking.go b/backend/core/models/locking.go index 9510e24f345..ff2b1cdd55f 100644 --- a/backend/core/models/locking.go +++ b/backend/core/models/locking.go @@ -19,7 +19,7 @@ package models import "time" -// LockingHistory is desgned for preventing mutiple delake instances from sharing the same database which may cause +// LockingHistory is desgned for preventing multiple delake instances from sharing the same database which may cause // problems like #3537, #3466. It works by the following step: // // 1. Each devlake insert a record to this table whie `Succeeded=false` diff --git a/backend/core/models/migrationscripts/20260217_modify_cicd_deployments_to_text.go b/backend/core/models/migrationscripts/20260217_modify_cicd_deployments_to_text.go new file mode 100644 index 00000000000..348099cea94 --- /dev/null +++ b/backend/core/models/migrationscripts/20260217_modify_cicd_deployments_to_text.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +var _ plugin.MigrationScript = (*modifyCicdDeploymentsToText)(nil) + +type modifyCicdDeploymentsToText struct{} + +type cicdDeployment20260217 struct { + Name string +} + +func (cicdDeployment20260217) TableName() string { + return "cicd_deployments" +} + +func (script *modifyCicdDeploymentsToText) Up(basicRes context.BasicRes) errors.Error { + // cicd_deployments.name might be text, we ought to change the type + // for the column from `varchar(255)` to `text` + db := basicRes.GetDal() + return migrationhelper.ChangeColumnsType[cicdDeployment20260217]( + basicRes, + script, + cicdDeployment20260217{}.TableName(), + []string{"name"}, + func(tmpColumnParams []interface{}) errors.Error { + return db.UpdateColumn( + &cicdDeployment20260217{}, + "name", + dal.DalClause{Expr: " ? ", Params: tmpColumnParams}, + dal.Where("? != '' ", tmpColumnParams...), + ) + }, + ) +} + +func (*modifyCicdDeploymentsToText) Version() uint64 { + return 20260217145125 +} + +func (*modifyCicdDeploymentsToText) Name() string { + return "modify cicd_deployments name from varchar to text" +} diff --git a/backend/core/models/migrationscripts/20260313_add_is_invalid_to_qa_test_case_executions.go b/backend/core/models/migrationscripts/20260313_add_is_invalid_to_qa_test_case_executions.go new file mode 100644 index 00000000000..ce7656e1e4c --- /dev/null +++ b/backend/core/models/migrationscripts/20260313_add_is_invalid_to_qa_test_case_executions.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addIsInvalidToQaTestCaseExecution)(nil) + +type qaTestCaseExecution20260313 struct { + IsInvalid bool +} + +func (qaTestCaseExecution20260313) TableName() string { + return "qa_test_case_executions" +} + +type addIsInvalidToQaTestCaseExecution struct{} + +func (*addIsInvalidToQaTestCaseExecution) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.AutoMigrate(&qaTestCaseExecution20260313{}); err != nil { + return err + } + return nil +} + +func (*addIsInvalidToQaTestCaseExecution) Version() uint64 { + return 20260313100000 +} + +func (*addIsInvalidToQaTestCaseExecution) Name() string { + return "add is_invalid to qa_test_case_executions" +} diff --git a/backend/core/models/migrationscripts/20260317_increase_cq_issues_project_key_length.go b/backend/core/models/migrationscripts/20260317_increase_cq_issues_project_key_length.go new file mode 100644 index 00000000000..223a2b6a345 --- /dev/null +++ b/backend/core/models/migrationscripts/20260317_increase_cq_issues_project_key_length.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*increaseCqIssuesProjectKeyLength)(nil) + +type increaseCqIssuesProjectKeyLength struct{} + +func (script *increaseCqIssuesProjectKeyLength) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.ModifyColumnType("cq_issues", "project_key", "varchar(500)"); err != nil { + return err + } + return db.ModifyColumnType("cq_file_metrics", "project_key", "varchar(500)") +} + +func (*increaseCqIssuesProjectKeyLength) Version() uint64 { + return 20260317000000 +} + +func (*increaseCqIssuesProjectKeyLength) Name() string { + return "increase cq_issues and cq_file_metrics project_key length to 500" +} diff --git a/backend/core/models/migrationscripts/20260426_add_auth_sessions.go b/backend/core/models/migrationscripts/20260426_add_auth_sessions.go new file mode 100644 index 00000000000..7e6eab8aacd --- /dev/null +++ b/backend/core/models/migrationscripts/20260426_add_auth_sessions.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +var _ plugin.MigrationScript = (*addAuthSessions)(nil) + +type authSession20260426 struct { + Jti string `gorm:"primaryKey;type:varchar(36)"` + Sub string `gorm:"type:varchar(255);index"` + Email string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + IssuedAt time.Time `gorm:"not null"` + ExpiresAt time.Time `gorm:"index;not null"` + RevokedAt *time.Time `gorm:"index"` + LastSeenAt time.Time +} + +func (authSession20260426) TableName() string { + return "auth_sessions" +} + +type addAuthSessions struct{} + +func (*addAuthSessions) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, new(authSession20260426)) +} + +func (*addAuthSessions) Version() uint64 { + return 20260426000001 +} + +func (*addAuthSessions) Name() string { + return "add auth_sessions table for session revocation" +} diff --git a/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go new file mode 100644 index 00000000000..94d965d6295 --- /dev/null +++ b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeIssueComponentToText)(nil) + +type changeIssueComponentToText struct{} + +func (*changeIssueComponentToText) Up(basicRes context.BasicRes) errors.Error { + // The 20240813 migration targeted the non-existent plural "components" column. + return basicRes.GetDal().ModifyColumnType("issues", "component", "text") +} + +func (*changeIssueComponentToText) Version() uint64 { + return 20260629120000 +} + +func (*changeIssueComponentToText) Name() string { + return "change issues.component type to text" +} diff --git a/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go new file mode 100644 index 00000000000..482c42b5001 --- /dev/null +++ b/backend/core/models/migrationscripts/20260629_change_issue_component_to_text_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" +) + +type modifyColumnTypeCall struct { + tableName string + columnName string + columnType string +} + +type recordingDal struct { + dal.Dal + call *modifyColumnTypeCall +} + +func (d *recordingDal) ModifyColumnType(tableName string, columnName string, columnType string) errors.Error { + d.call = &modifyColumnTypeCall{tableName, columnName, columnType} + return nil +} + +type basicResWithDal struct { + context.BasicRes + database dal.Dal +} + +func (r *basicResWithDal) GetDal() dal.Dal { + return r.database +} + +func TestChangeIssueComponentToText(t *testing.T) { + database := new(recordingDal) + script := new(changeIssueComponentToText) + + if err := script.Up(&basicResWithDal{database: database}); err != nil { + t.Fatalf("migration failed: %v", err) + } + + want := &modifyColumnTypeCall{"issues", "component", "text"} + if database.call == nil || *database.call != *want { + t.Fatalf("ModifyColumnType call = %#v, want %#v", database.call, want) + } + if script.Version() != 20260629120000 { + t.Fatalf("Version() = %d, want 20260629120000", script.Version()) + } + if script.Name() != "change issues.component type to text" { + t.Fatalf("Name() = %q, want %q", script.Name(), "change issues.component type to text") + } + + found := false + for _, registeredScript := range All() { + if registeredScript.Version() == script.Version() { + found = true + break + } + } + if !found { + t.Fatalf("migration version %d is not registered", script.Version()) + } +} diff --git a/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go b/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go new file mode 100644 index 00000000000..b84d1d9ce1b --- /dev/null +++ b/backend/core/models/migrationscripts/20260701_change_cq_issue_code_blocks_component_to_text.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeCqIssueCodeBlocksComponentToText)(nil) + +type changeCqIssueCodeBlocksComponentToText struct{} + +func (script *changeCqIssueCodeBlocksComponentToText) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.DropIndexes("cq_issue_code_blocks", "idx_cq_issue_code_blocks_component"); err != nil { + return err + } + return db.ModifyColumnType("cq_issue_code_blocks", "component", "text") +} + +func (*changeCqIssueCodeBlocksComponentToText) Version() uint64 { + return 20260701000000 +} + +func (*changeCqIssueCodeBlocksComponentToText) Name() string { + return "change cq_issue_code_blocks.component type to text" +} diff --git a/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go b/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go new file mode 100644 index 00000000000..de66ae82914 --- /dev/null +++ b/backend/core/models/migrationscripts/20260707_add_cq_project_metrics_history.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCqProjectMetricsHistory struct{} + +func (u *addCqProjectMetricsHistory) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.CqProjectMetricsHistory{}, + ) +} + +func (*addCqProjectMetricsHistory) Version() uint64 { + return 20260707153201 +} + +func (*addCqProjectMetricsHistory) Name() string { + return "add cq_project_metrics_history domain table" +} diff --git a/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go b/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go new file mode 100644 index 00000000000..934627368e2 --- /dev/null +++ b/backend/core/models/migrationscripts/archived/cq_project_metrics_history.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import "time" + +type CqProjectMetricsHistory struct { + DomainEntity + ProjectKey string `gorm:"index;type:varchar(500)"` + AnalysisDate time.Time `gorm:"index"` + Coverage *float64 + Ncloc *int + Bugs *int + ReliabilityRating string `gorm:"type:varchar(5)"` + CodeSmells *int + SqaleRating string `gorm:"type:varchar(5)"` + Complexity *int + CognitiveComplexity *int + Vulnerabilities *int + SecurityRating string `gorm:"type:varchar(5)"` + SecurityHotspots *int + DuplicatedLinesDensity *float64 +} + +func (CqProjectMetricsHistory) TableName() string { + return "cq_project_metrics_history" +} diff --git a/backend/core/models/migrationscripts/register.go b/backend/core/models/migrationscripts/register.go index 5e73221d11f..17557309705 100644 --- a/backend/core/models/migrationscripts/register.go +++ b/backend/core/models/migrationscripts/register.go @@ -136,11 +136,18 @@ func All() []plugin.MigrationScript { new(addCqIssueImpacts), new(addDueDateToIssues), new(createQaTables), + new(addIsInvalidToQaTestCaseExecution), new(increaseCqIssueComponentLength), new(extendFieldSizeForCq), new(addIssueFixVerion), new(addPipelinePriority), new(fixNullPriority), new(convertPrMetricsToMilliseconds), + new(modifyCicdDeploymentsToText), + new(increaseCqIssuesProjectKeyLength), + new(addAuthSessions), + new(changeIssueComponentToText), + new(changeCqIssueCodeBlocksComponentToText), + new(addCqProjectMetricsHistory), } } diff --git a/backend/core/plugin/plugin_blueprint.go b/backend/core/plugin/plugin_blueprint.go index 492e54d43dd..35c36b36b25 100644 --- a/backend/core/plugin/plugin_blueprint.go +++ b/backend/core/plugin/plugin_blueprint.go @@ -78,9 +78,9 @@ type DataSourcePluginBlueprintV200 interface { // BlueprintConnectionV200 contains the pluginName/connectionId and related Scopes, // MetricPluginBlueprintV200 is similar to the DataSourcePluginBlueprintV200 -// but for Metric Plugin, take dora as an example, it doens't have any scope, +// but for Metric Plugin, take dora as an example, it doesn't have any scope, // nor does it produce any, however, it does require other plugin to be -// executed beforehand, like calcuating refdiff before it can connect PR to the +// executed beforehand, like calculating refdiff before it can connect PR to the // right Deployment keep in mind it would be called IFF the plugin was enabled // for the project. type MetricPluginBlueprintV200 interface { diff --git a/backend/core/plugin/plugin_task.go b/backend/core/plugin/plugin_task.go index 136f4f17eb8..e3eea470a81 100644 --- a/backend/core/plugin/plugin_task.go +++ b/backend/core/plugin/plugin_task.go @@ -53,6 +53,7 @@ type ExecContext interface { GetData() interface{} SetProgress(current int, total int) IncProgress(quantity int) + GetProgress() int } // SubTaskContext This interface define all resources that needed for subtask execution @@ -106,7 +107,7 @@ type SubTaskMeta struct { Dependencies []*SubTaskMeta DependencyTables []string ProductTables []string - ForceRunOnResume bool // Should a subtask be ran dispite it was finished before + ForceRunOnResume bool // Should a subtask be ran despite it was finished before } // PluginTask Implement this interface to let framework run tasks for you diff --git a/backend/core/plugin/plugin_utils.go b/backend/core/plugin/plugin_utils.go index dd0f29ca4aa..ac8e2090b58 100644 --- a/backend/core/plugin/plugin_utils.go +++ b/backend/core/plugin/plugin_utils.go @@ -21,9 +21,11 @@ import ( "bytes" "crypto/aes" "crypto/cipher" + "crypto/rand" "crypto/sha256" "encoding/base64" "fmt" + "io" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/utils" @@ -31,8 +33,11 @@ import ( const EncodeKeyEnvStr = "ENCRYPTION_SECRET" -// TODO: maybe move encryption/decryption into helper? -// AES + Base64 encryption using ENCRYPTION_SECRET in .env as key +// gcmNonceSize is the standard nonce size for AES-GCM. +const gcmNonceSize = 12 + +// Encrypt AES-GCM encrypts plaintext using ENCRYPTION_SECRET, then base64-encodes the result. +// The output format is: base64(nonce || ciphertext || tag). func Encrypt(encryptionSecret, plainText string) (string, errors.Error) { // add suffix to the data part inputBytes := append([]byte(plainText), 123, 110, 100, 100, 116, 102, 125) @@ -45,7 +50,8 @@ func Encrypt(encryptionSecret, plainText string) (string, errors.Error) { return base64.StdEncoding.EncodeToString(output), nil } -// Base64 + AES decryption using ENCRYPTION_SECRET in .env as key +// Decrypt base64-decodes then AES-GCM decrypts ciphertext using ENCRYPTION_SECRET. +// For backward compatibility, it also attempts AES-CBC decryption if the data looks like legacy format. func Decrypt(encryptionSecret, encryptedText string) (string, errors.Error) { // when encryption key is not set if encryptionSecret == "" { @@ -98,41 +104,59 @@ func PKCS7UnPadding(origData []byte) []byte { return origData[:(length - unpadding)] } -// AesEncrypt AES encryption, CBC +// AesEncrypt AES-256-GCM encrypts origData using key. +// The returned bytes are: nonce (12 bytes) || ciphertext || tag. func AesEncrypt(origData, key []byte) ([]byte, errors.Error) { - // data alignment fill and encryption sha256Key := sha256.Sum256(key) - key = sha256Key[:] - block, err := aes.NewCipher(key) + block, err := aes.NewCipher(sha256Key[:]) if err != nil { return nil, errors.Convert(err) } - // data alignment fill and encryption - blockSize := block.BlockSize() - origData = PKCS7Padding(origData, blockSize) - blockMode := cipher.NewCBCEncrypter(block, key[:blockSize]) - crypted := make([]byte, len(origData)) - blockMode.CryptBlocks(crypted, origData) - return crypted, nil + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Convert(err) + } + nonce := make([]byte, gcmNonceSize) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errors.Convert(err) + } + ciphertext := gcm.Seal(nonce, nonce, origData, nil) + return ciphertext, nil } -// AesDecrypt AES decryption +// AesDecrypt decrypts crypted data using key. +// It first tries AES-256-GCM (expects a 12-byte nonce prefix). +// If that fails and the data length is a multiple of the AES block size (legacy CBC format), +// it falls back to AES-256-CBC for backward compatibility. func AesDecrypt(crypted, key []byte) ([]byte, errors.Error) { - // Uniformly use sha256 to process as 32-bit Byte (256-bit bit) sha256Key := sha256.Sum256(key) - key = sha256Key[:] - block, err := aes.NewCipher(key) + block, err := aes.NewCipher(sha256Key[:]) if err != nil { return nil, errors.Convert(err) } - // Get the block size and check whether the ciphertext length is legal blockSize := block.BlockSize() + + // Try GCM first if the data is long enough to contain a nonce. + if len(crypted) >= gcmNonceSize+blockSize { + gcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Convert(err) + } + nonce := crypted[:gcmNonceSize] + ciphertext := crypted[gcmNonceSize:] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err == nil { + return plaintext, nil + } + // GCM decryption failed; fall through to try legacy CBC. + } + + // Legacy CBC fallback. if len(crypted)%blockSize != 0 { return nil, errors.Default.New(fmt.Sprintf("The length of the data to be decrypted is [%d], so cannot match the required block size [%d]", len(crypted), blockSize)) } - // Decrypt and unalign data - blockMode := cipher.NewCBCDecrypter(block, key[:blockSize]) + blockMode := cipher.NewCBCDecrypter(block, sha256Key[:blockSize]) origData := make([]byte, len(crypted)) blockMode.CryptBlocks(origData, crypted) origData = PKCS7UnPadding(origData) diff --git a/backend/core/plugin/plugin_utils_test.go b/backend/core/plugin/plugin_utils_test.go index b42a4c57b48..9a8f516adad 100644 --- a/backend/core/plugin/plugin_utils_test.go +++ b/backend/core/plugin/plugin_utils_test.go @@ -18,8 +18,13 @@ limitations under the License. package plugin import ( + "crypto/aes" + "crypto/cipher" + "crypto/sha256" + "encoding/base64" "testing" + "github.com/apache/incubator-devlake/core/errors" "github.com/stretchr/testify/assert" ) @@ -70,3 +75,67 @@ func TestEncode(t *testing.T) { }) } } + +func TestGCMEncDec(t *testing.T) { + TestStr := "The string for testing" + encryptionSecret, _ := RandomEncryptionSecret() + + // Encrypt with the new GCM format. + newCiphertext, err := Encrypt(encryptionSecret, TestStr) + assert.Empty(t, err) + + // Decrypt the new format. + decodedNew, err := Decrypt(encryptionSecret, newCiphertext) + assert.Empty(t, err) + assert.Equal(t, TestStr, decodedNew) + + // Ensure two encryptions of the same plaintext produce different ciphertexts (random nonce). + newCiphertext2, err := Encrypt(encryptionSecret, TestStr) + assert.Empty(t, err) + assert.NotEqual(t, newCiphertext, newCiphertext2) +} + +// AesEncrypt AES encryption, CBC +func oldAesEncrypt(origData, key []byte) ([]byte, errors.Error) { + // data alignment fill and encryption + sha256Key := sha256.Sum256(key) + key = sha256Key[:] + block, err := aes.NewCipher(key) + if err != nil { + return nil, errors.Convert(err) + } + // data alignment fill and encryption + blockSize := block.BlockSize() + origData = PKCS7Padding(origData, blockSize) + blockMode := cipher.NewCBCEncrypter(block, key[:blockSize]) + crypted := make([]byte, len(origData)) + blockMode.CryptBlocks(crypted, origData) + return crypted, nil +} + +func oldEncrypt(encryptionSecret, plainText string) (string, errors.Error) { + // add suffix to the data part + inputBytes := append([]byte(plainText), 123, 110, 100, 100, 116, 102, 125) + // perform encryption + output, err := oldAesEncrypt(inputBytes, []byte(encryptionSecret)) + if err != nil { + return plainText, err + } + // Return the result after Base64 processing + return base64.StdEncoding.EncodeToString(output), nil +} + +func TestBackwardCompatibility(t *testing.T) { + TestStr := "The string for testing" + encryptionSecret, _ := RandomEncryptionSecret() + + // Encrypt with the new GCM format. + newCiphertext, err := oldEncrypt(encryptionSecret, TestStr) + assert.Empty(t, err) + + // Decrypt the new format. + decodedNew, err := Decrypt(encryptionSecret, newCiphertext) + assert.Empty(t, err) + assert.Equal(t, TestStr, decodedNew) + +} diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go index 2b848ea9bbd..961725bce79 100644 --- a/backend/core/runner/run_task.go +++ b/backend/core/runner/run_task.go @@ -336,7 +336,7 @@ func RunPluginSubTasks( logger.Info("executing subtask %s", subtaskMeta.Name) start := time.Now() err = runSubtask(basicRes, subtaskCtx, task.ID, subtaskNumber, subtaskMeta.EntryPoint) - logger.Info("subtask %s finished in %d ms", subtaskMeta.Name, time.Since(start).Milliseconds()) + logger.Info("subtask %s finished in %d ms, records processed: %d", subtaskMeta.Name, time.Since(start).Milliseconds(), subtaskCtx.GetProgress()) if err != nil { err = errors.SubtaskErr.Wrap(err, fmt.Sprintf("subtask %s ended unexpectedly", subtaskMeta.Name), errors.WithData(&subtaskMeta)) logger.Error(err, "") diff --git a/backend/core/utils/network_helper.go b/backend/core/utils/network_helper.go index a781b80b723..aba08452704 100644 --- a/backend/core/utils/network_helper.go +++ b/backend/core/utils/network_helper.go @@ -19,9 +19,10 @@ package utils import ( "fmt" - "github.com/apache/incubator-devlake/core/errors" "net" "time" + + "github.com/apache/incubator-devlake/core/errors" ) // CheckDNS FIXME ... @@ -38,7 +39,7 @@ func CheckDNS(domain string) errors.Error { // CheckNetwork FIXME ... func CheckNetwork(host, port string, timeout time.Duration) errors.Error { - var target = fmt.Sprintf("%s:%s", host, port) + var target = net.JoinHostPort(host, port) _, err := net.DialTimeout("tcp", target, timeout) if err != nil { return errors.Convert(err) diff --git a/backend/core/utils/network_helper_test.go b/backend/core/utils/network_helper_test.go index 86f76fbb1b2..a46d4096b26 100644 --- a/backend/core/utils/network_helper_test.go +++ b/backend/core/utils/network_helper_test.go @@ -53,7 +53,7 @@ func TestResolvePort(t *testing.T) { } _, err = ResolvePort("", "rabbitmq") if err == nil { - t.Errorf("Expected error %s, Got nil", "schema not fount") + t.Errorf("Expected error %s, Got nil", "schema not found") } _, err = ResolvePort("", "") if err == nil { diff --git a/backend/go.mod b/backend/go.mod index a08cbd3ecdf..54a69baa4f5 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -1,6 +1,6 @@ module github.com/apache/incubator-devlake -go 1.20 +go 1.26 require ( github.com/aws/aws-sdk-go v1.55.6 @@ -16,7 +16,6 @@ require ( github.com/lib/pq v1.10.2 github.com/libgit2/git2go/v33 v33.0.6 github.com/magiconair/properties v1.8.5 - github.com/merico-dev/graphql v0.0.0-20240807070533-1cafa544cd5d github.com/mitchellh/mapstructure v1.5.0 github.com/panjf2000/ants/v2 v2.4.6 github.com/robfig/cron/v3 v3.0.0 @@ -33,7 +32,7 @@ require ( github.com/viant/afs v1.16.0 golang.org/x/crypto v0.26.0 // indirect golang.org/x/exp v0.0.0-20221028150844-83b7d23a625f - golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 + golang.org/x/oauth2 v0.13.0 golang.org/x/sync v0.8.0 gorm.io/datatypes v1.0.1 gorm.io/driver/mysql v1.5.1 @@ -43,8 +42,10 @@ require ( require ( github.com/chainguard-dev/git-urls v1.0.2 + github.com/coreos/go-oidc/v3 v3.9.0 github.com/go-sql-driver/mysql v1.7.1 github.com/golang-jwt/jwt/v5 v5.0.0-rc.1 + github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac github.com/rogpeppe/go-internal v1.11.0 golang.org/x/mod v0.17.0 ) @@ -70,6 +71,7 @@ require ( github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.1 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect @@ -79,7 +81,7 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect @@ -121,8 +123,8 @@ require ( golang.org/x/sys v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 8adaf365f49..3bdf720600d 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -90,6 +90,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo= +github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -141,6 +143,8 @@ github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXY github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= +github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -205,8 +209,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -388,8 +393,8 @@ github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APP github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ= github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI= -github.com/merico-dev/graphql v0.0.0-20240807070533-1cafa544cd5d h1:FpP+YRQudZtnrnIaFvVc87D/WVI7tWi0hBrnRCY8wmQ= -github.com/merico-dev/graphql v0.0.0-20240807070533-1cafa544cd5d/go.mod h1:dcDqG8HXVtfEhTCipFMa0Q+RTKTtDKIO2vJt+JVzHEQ= +github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac h1:J3wm8OohrRkYf/+Zy5GxqvhzcP2qHGJRduxTUUbBMdw= +github.com/merico-ai/graphql v0.0.0-20260206020408-b7fd267bcfac/go.mod h1:4WSOgvd/Sv6eLKJE6fvrXlmxePBDGhbwRYKEBeqJr8g= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -658,8 +663,9 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -755,6 +761,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -856,8 +863,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -931,8 +939,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/backend/helpers/e2ehelper/data_flow_tester.go b/backend/helpers/e2ehelper/data_flow_tester.go index 0759571d203..784238fd242 100644 --- a/backend/helpers/e2ehelper/data_flow_tester.go +++ b/backend/helpers/e2ehelper/data_flow_tester.go @@ -119,7 +119,7 @@ func NewDataFlowTester(t *testing.T, pluginName string, pluginMeta plugin.Plugin cfg.Set(`DB_URL`, cfg.GetString(`E2E_DB_URL`)) db, err := runner.NewGormDb(cfg, logruslog.Global) if err != nil { - // if here fail with error `acces denied for user` you need to create database by your self as follow command + // if here fail with error `access denied for user` you need to create database by your self as follow command // create databases lake_test; // grant all on lake_test.* to 'merico'@'%'; panic(err) diff --git a/backend/helpers/migrationhelper/migrationhelper.go b/backend/helpers/migrationhelper/migrationhelper.go index 24eb537a793..691413b1a56 100644 --- a/backend/helpers/migrationhelper/migrationhelper.go +++ b/backend/helpers/migrationhelper/migrationhelper.go @@ -32,7 +32,7 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) -// AutoMigrateTables runs AutoMigrate for muliple tables +// AutoMigrateTables runs AutoMigrate for multiple tables func AutoMigrateTables(basicRes context.BasicRes, dst ...interface{}) errors.Error { db := basicRes.GetDal() for _, entity := range dst { @@ -74,14 +74,14 @@ func ChangeColumnsType[D any]( err = db.AutoMigrate(new(D), dal.From(tableName)) if err != nil { - return errors.Default.Wrap(err, "AutoMigrate for Add Colume Error") + return errors.Default.Wrap(err, "AutoMigrate for Add Column Error") } defer func() { if err != nil { err1 := db.DropColumns(tableName, columns...) if err1 != nil { - err = errors.Default.Wrap(err, fmt.Sprintf("RollBack by DropColume failed.Relevant data needs to be repaired manually.%s", err1.Error())) + err = errors.Default.Wrap(err, fmt.Sprintf("RollBack by DropColumn failed.Relevant data needs to be repaired manually.%s", err1.Error())) } } }() diff --git a/backend/helpers/migrationhelper/migrationhelper_test.go b/backend/helpers/migrationhelper/migrationhelper_test.go index 5fe70d52b35..7962100e453 100644 --- a/backend/helpers/migrationhelper/migrationhelper_test.go +++ b/backend/helpers/migrationhelper/migrationhelper_test.go @@ -132,7 +132,7 @@ func TestTransformTable(t *testing.T) { assert.Equal(t, dts[2].Id, "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca1357ef3d346f24f386216563752b0c447a35c041e0b7143f929dc4de27742e3307") }).Return(nil).Once() - // for Primarykey autoincrement cheking + // for Primarykey autoincrement checking mockDal.On("GetColumns", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName := args.Get(0).(dal.Tabler).TableName() assert.Equal(t, tableName, TestTableNameSrc) @@ -188,7 +188,7 @@ func TestTransformTable_RollBack(t *testing.T) { assert.NotEqual(t, oldname, tmpname) }).Return(nil).Once() - // checking if Rename and Drop RollBack working with rigth table + // checking if Rename and Drop RollBack working with right table mockDal.On("RenameTable", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tmpname, ok := args.Get(0).(string) assert.Equal(t, ok, true) @@ -203,7 +203,7 @@ func TestTransformTable_RollBack(t *testing.T) { assert.Equal(t, oldname, TestTableNameSrc) }).Return(nil).Once() - // for Primarykey autoincrement cheking + // for Primarykey autoincrement checking mockDal.On("GetColumns", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName := args.Get(0).(dal.Tabler).TableName() assert.Equal(t, tableName, TestTableNameSrc) @@ -296,7 +296,7 @@ func TestCopyTableColumns(t *testing.T) { assert.Equal(t, dts[2].Id, "fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca1357ef3d346f24f386216563752b0c447a35c041e0b7143f929dc4de27742e3307") }).Return(nil).Once() - // for Primarykey autoincrement cheking + // for Primarykey autoincrement checking mockDal.On("GetColumns", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName := args.Get(0).(dal.Tabler).TableName() assert.Equal(t, tableName, TestTableNameSrc) @@ -352,7 +352,7 @@ func TestCopyTableColumns_RollBack(t *testing.T) { assert.NotEqual(t, oldname, tmpname) }).Return(nil).Once() - // checking if Rename and Drop RollBack working with rigth table + // checking if Rename and Drop RollBack working with right table mockDal.On("RenameTable", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tmpname, ok := args.Get(0).(string) assert.Equal(t, ok, true) @@ -367,7 +367,7 @@ func TestCopyTableColumns_RollBack(t *testing.T) { assert.Equal(t, oldname, TestTableNameSrc) }).Return(nil).Once() - // for Primarykey autoincrement cheking + // for Primarykey autoincrement checking mockDal.On("GetColumns", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName := args.Get(0).(dal.Tabler).TableName() assert.Equal(t, tableName, TestTableNameSrc) @@ -521,7 +521,7 @@ func TestTransformColumns_RollBack(t *testing.T) { assert.NotEqual(t, columnName, tmpColumnName) }).Return(nil).Once() - // checking if Rename and Drop RollBack working with rigth table + // checking if Rename and Drop RollBack working with right table mockDal.On("RenameColumn", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName, ok := args.Get(0).(string) assert.Equal(t, ok, true) @@ -634,7 +634,7 @@ func TestChangeColumnsType_Rollback(t *testing.T) { assert.NotEqual(t, columnName, tmpColumnName) }).Return(nil).Once() - // checking if Rename and Drop RollBack working with rigth table + // checking if Rename and Drop RollBack working with right table mockDal.On("RenameColumn", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { tableName, ok := args.Get(0).(string) assert.Equal(t, ok, true) diff --git a/backend/helpers/oidchelper/authorization.go b/backend/helpers/oidchelper/authorization.go new file mode 100644 index 00000000000..c740666a275 --- /dev/null +++ b/backend/helpers/oidchelper/authorization.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import "strings" + +func (c *Config) IsUserAllowed(email string) bool { + if len(c.AllowEmails) == 0 && + len(c.AllowDomains) == 0 { + return true + } + + email = strings.ToLower(strings.TrimSpace(email)) + + if _, ok := c.AllowEmails[email]; ok { + return true + } + + _, domain, ok := strings.Cut(email, "@") + if ok { + if _, ok := c.AllowDomains[domain]; ok { + return true + } + } + + return false +} diff --git a/backend/helpers/oidchelper/authorization_test.go b/backend/helpers/oidchelper/authorization_test.go new file mode 100644 index 00000000000..e73a32e92d3 --- /dev/null +++ b/backend/helpers/oidchelper/authorization_test.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import "testing" + +func TestIsUserAllowed(t *testing.T) { + cases := []struct { + name string + cfg Config + email string + want bool + }{ + { + name: "no restrictions", + cfg: Config{}, + email: "user@example.com", + want: true, + }, + { + name: "allowed email", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "user@example.com", + want: true, + }, + { + name: "blocked email", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "other@example.com", + want: false, + }, + { + name: "allowed domain", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@example.com", + want: true, + }, + { + name: "blocked domain", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@other.com", + want: false, + }, + { + name: "email case insensitive", + cfg: Config{ + AllowEmails: map[string]struct{}{ + "user@example.com": {}, + }, + }, + email: "USER@example.com", + want: true, + }, + { + name: "domain case insensitive", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "user@EXAMPLE.COM", + want: true, + }, + { + name: "invalid email", + cfg: Config{ + AllowDomains: map[string]struct{}{ + "example.com": {}, + }, + }, + email: "not-an-email", + want: false, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := tc.cfg.IsUserAllowed(tc.email); got != tc.want { + t.Errorf( + "IsUserAllowed(%q) = %v, want %v", + tc.email, + got, + tc.want, + ) + } + }) + } +} diff --git a/backend/helpers/oidchelper/config.go b/backend/helpers/oidchelper/config.go new file mode 100644 index 00000000000..9d1b8ae47c2 --- /dev/null +++ b/backend/helpers/oidchelper/config.go @@ -0,0 +1,255 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "fmt" + "sort" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/context" +) + +const ( + SessionCookieName = "devlake_session" + StateCookieName = "devlake_oauth_state" + // CSRFCookieName is intentionally readable by JS so axios can echo it + // back as the X-CSRF-Token header (double-submit cookie pattern). + CSRFCookieName = "devlake_csrf" + CSRFHeaderName = "X-CSRF-Token" + + // State cookie max-age. Covers a slow user typing 2FA at the IdP. + StateCookieMaxAge = 10 * time.Minute + + defaultSessionTTL = 8 * time.Hour + defaultScopes = "openid,profile,email" +) + +// ProviderConfig holds everything specific to one OIDC IdP. Multiple +// instances live in Config.Providers, keyed by Name. +type ProviderConfig struct { + Name string + IssuerURL string + ClientID string + ClientSecret string + RedirectURL string + Scopes []string + DisplayName string + + // UseWorkloadIdentity authenticates the code exchange with an Azure + // Workload Identity federated assertion (read from the SA token file) + // instead of ClientSecret. Entra-specific. + UseWorkloadIdentity bool +} + +// Config is the typed view of the auth-related env vars. Build it once at boot. +type Config struct { + AuthEnabled bool + + OIDCEnabled bool + Providers map[string]*ProviderConfig + LogoutRedirect bool + + // Optional OIDC authorization restrictions. + // Empty means no restriction. + AllowEmails map[string]struct{} + AllowDomains map[string]struct{} + + SessionSecret []byte + SessionTTL time.Duration + + CookieDomain string + CookieSecure bool +} + +// ProviderNames returns the configured provider names in stable order. +func (c *Config) ProviderNames() []string { + if c == nil { + return nil + } + out := make([]string, 0, len(c.Providers)) + for name := range c.Providers { + out = append(out, name) + } + sort.Strings(out) + return out +} + +// LoadConfig reads auth env vars via Viper and validates required fields. +// AUTH_ENABLED defaults to false unless it is explicitly set to true. +func LoadConfig(basicRes context.BasicRes) (*Config, error) { + cfg := basicRes.GetConfigReader() + + authEnabled := false + if cfg.IsSet("AUTH_ENABLED") { + authEnabled = cfg.GetBool("AUTH_ENABLED") + } + if !authEnabled { + return &Config{AuthEnabled: false}, nil + } + + oidcEnabled := cfg.GetBool("OIDC_ENABLED") + sessionSecret := strings.TrimSpace(cfg.GetString("SESSION_SECRET")) + if oidcEnabled { + if sessionSecret == "" { + return nil, fmt.Errorf("OIDC_ENABLED=true but SESSION_SECRET is not set") + } + if len(sessionSecret) < 32 { + return nil, fmt.Errorf("SESSION_SECRET must be at least 32 bytes") + } + } else if sessionSecret != "" && len(sessionSecret) < 32 { + return nil, fmt.Errorf("SESSION_SECRET must be at least 32 bytes") + } + + ttl := defaultSessionTTL + if v := strings.TrimSpace(cfg.GetString("SESSION_TTL")); v != "" { + parsed, err := time.ParseDuration(v) + if err != nil { + return nil, fmt.Errorf("invalid SESSION_TTL %q: %w", v, err) + } + ttl = parsed + } + + cookieSecure := true + if cfg.IsSet("COOKIE_SECURE") { + cookieSecure = cfg.GetBool("COOKIE_SECURE") + } + + out := &Config{ + AuthEnabled: true, + OIDCEnabled: oidcEnabled, + Providers: map[string]*ProviderConfig{}, + LogoutRedirect: cfg.GetBool("OIDC_LOGOUT_REDIRECT"), + SessionSecret: []byte(sessionSecret), + SessionTTL: ttl, + CookieDomain: strings.TrimSpace(cfg.GetString("COOKIE_DOMAIN")), + CookieSecure: cookieSecure, + AllowEmails: parseStringSet(cfg.GetString("OIDC_ALLOW_EMAILS")), + AllowDomains: parseStringSet(cfg.GetString("OIDC_ALLOW_DOMAINS")), + } + + if !out.OIDCEnabled { + return out, nil + } + + names := parseProviderNames(cfg.GetString("OIDC_PROVIDERS")) + if len(names) == 0 { + return nil, fmt.Errorf("OIDC_ENABLED=true but OIDC_PROVIDERS is empty") + } + for _, name := range names { + prefix := "OIDC_" + strings.ToUpper(name) + "_" + p, err := loadProviderConfig(cfg, name, prefix) + if err != nil { + return nil, err + } + out.Providers[name] = p + } + return out, nil +} + +func loadProviderConfig(cfg config.ConfigReader, name, prefix string) (*ProviderConfig, error) { + p := &ProviderConfig{ + Name: name, + IssuerURL: strings.TrimSpace(cfg.GetString(prefix + "ISSUER_URL")), + ClientID: strings.TrimSpace(cfg.GetString(prefix + "CLIENT_ID")), + ClientSecret: strings.TrimSpace(cfg.GetString(prefix + "CLIENT_SECRET")), + RedirectURL: strings.TrimSpace(cfg.GetString(prefix + "REDIRECT_URL")), + Scopes: parseScopes(cfg.GetString(prefix + "SCOPES")), + DisplayName: valueOr(cfg.GetString(prefix+"DISPLAY_NAME"), name), + UseWorkloadIdentity: cfg.GetBool(prefix + "USE_WORKLOAD_IDENTITY"), + } + + missing := []string{} + if p.IssuerURL == "" { + missing = append(missing, prefix+"ISSUER_URL") + } + if p.ClientID == "" { + missing = append(missing, prefix+"CLIENT_ID") + } + // Workload Identity replaces the static client secret with a federated + // assertion read at exchange time, so the secret is optional in that mode. + if p.ClientSecret == "" && !p.UseWorkloadIdentity { + missing = append(missing, prefix+"CLIENT_SECRET") + } + if p.RedirectURL == "" { + missing = append(missing, prefix+"REDIRECT_URL") + } + if len(missing) > 0 { + return nil, fmt.Errorf("provider %q missing required env vars: %s", name, strings.Join(missing, ", ")) + } + return p, nil +} + +func parseProviderNames(raw string) []string { + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + seen := map[string]struct{}{} + for _, p := range parts { + n := strings.ToLower(strings.TrimSpace(p)) + if n == "" { + continue + } + if _, dup := seen[n]; dup { + continue + } + seen[n] = struct{}{} + out = append(out, n) + } + + return out +} + +func parseStringSet(raw string) map[string]struct{} { + out := make(map[string]struct{}) + + for _, v := range strings.Split(raw, ",") { + v = strings.ToLower(strings.TrimSpace(v)) + + if v == "" { + continue + } + + out[v] = struct{}{} + } + + return out +} + +func parseScopes(raw string) []string { + if strings.TrimSpace(raw) == "" { + raw = defaultScopes + } + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + s := strings.TrimSpace(p) + if s != "" { + out = append(out, s) + } + } + return out +} + +func valueOr(s, fallback string) string { + if strings.TrimSpace(s) == "" { + return fallback + } + return s +} diff --git a/backend/helpers/oidchelper/config_test.go b/backend/helpers/oidchelper/config_test.go new file mode 100644 index 00000000000..3fe92a1187a --- /dev/null +++ b/backend/helpers/oidchelper/config_test.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "reflect" + "testing" + + "github.com/spf13/viper" + + "github.com/apache/incubator-devlake/core/config" + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/log" +) + +func TestParseScopes(t *testing.T) { + cases := map[string][]string{ + "": {"openid", "profile", "email"}, + " ": {"openid", "profile", "email"}, + "openid": {"openid"}, + "openid,profile": {"openid", "profile"}, + " openid , profile , email ": {"openid", "profile", "email"}, + "openid,,email": {"openid", "email"}, + } + for input, want := range cases { + got := parseScopes(input) + if !reflect.DeepEqual(got, want) { + t.Errorf("parseScopes(%q) = %v, want %v", input, got, want) + } + } +} + +func TestValueOr(t *testing.T) { + if got := valueOr("hello", "fallback"); got != "hello" { + t.Errorf("valueOr(hello) = %q", got) + } + if got := valueOr("", "fallback"); got != "fallback" { + t.Errorf("valueOr(empty) = %q", got) + } + if got := valueOr(" ", "fallback"); got != "fallback" { + t.Errorf("valueOr(whitespace) = %q", got) + } +} + +func TestParseProviderNames(t *testing.T) { + cases := map[string][]string{ + "": {}, + " ": {}, + "entra": {"entra"}, + "entra,google": {"entra", "google"}, + " Entra , GOOGLE": {"entra", "google"}, + "entra,,google": {"entra", "google"}, + "entra,entra": {"entra"}, + "Entra,entra": {"entra"}, + } + for input, want := range cases { + got := parseProviderNames(input) + if len(got) == 0 && len(want) == 0 { + continue + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parseProviderNames(%q) = %v, want %v", input, got, want) + } + } +} + +func TestProviderNamesSorted(t *testing.T) { + c := &Config{ + Providers: map[string]*ProviderConfig{ + "google": {Name: "google"}, + "entra": {Name: "entra"}, + }, + } + if names := c.ProviderNames(); !reflect.DeepEqual(names, []string{"entra", "google"}) { + t.Errorf("ProviderNames = %v, want sorted [entra google]", names) + } +} + +type basicResStub struct { + cfg config.ConfigReader +} + +func (b basicResStub) GetConfigReader() config.ConfigReader { return b.cfg } +func (b basicResStub) GetConfig(string) string { return "" } +func (b basicResStub) GetLogger() log.Logger { return nil } +func (b basicResStub) NestedLogger(string) corectx.BasicRes { return nil } +func (b basicResStub) ReplaceLogger(log.Logger) corectx.BasicRes { + return nil +} +func (b basicResStub) GetDal() dal.Dal { return nil } + +func TestLoadConfigDefaultsAuthDisabled(t *testing.T) { + v := viper.New() + + cfg, err := LoadConfig(basicResStub{cfg: v}) + if err != nil { + t.Fatalf("LoadConfig returned error: %v", err) + } + if cfg.AuthEnabled { + t.Fatal("AuthEnabled should default to false when AUTH_ENABLED is unset") + } + if cfg.OIDCEnabled { + t.Fatal("OIDCEnabled should default to false") + } + if len(cfg.SessionSecret) != 0 { + t.Fatalf("SessionSecret = %q, want empty when OIDC is disabled", string(cfg.SessionSecret)) + } +} + +func TestLoadConfigRequiresSessionSecretForOIDC(t *testing.T) { + v := viper.New() + v.Set("AUTH_ENABLED", true) + v.Set("OIDC_ENABLED", true) + + if _, err := LoadConfig(basicResStub{cfg: v}); err == nil { + t.Fatal("LoadConfig should reject OIDC-enabled config without SESSION_SECRET") + } +} diff --git a/backend/helpers/oidchelper/cookie.go b/backend/helpers/oidchelper/cookie.go new file mode 100644 index 00000000000..c628d07e864 --- /dev/null +++ b/backend/helpers/oidchelper/cookie.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +func writeCookie(c *gin.Context, cfg *Config, name, value string, maxAge int, httpOnly bool) { + http.SetCookie(c.Writer, &http.Cookie{ + Name: name, + Value: value, + Path: "/", + Domain: cfg.CookieDomain, + MaxAge: maxAge, + HttpOnly: httpOnly, + Secure: cfg.CookieSecure, + SameSite: http.SameSiteLaxMode, + }) +} + +func SetSessionCookie(c *gin.Context, cfg *Config, jwt string) { + writeCookie(c, cfg, SessionCookieName, jwt, int(cfg.SessionTTL.Seconds()), true) +} + +func ClearSessionCookie(c *gin.Context, cfg *Config) { + writeCookie(c, cfg, SessionCookieName, "", -1, true) +} + +func SetStateCookie(c *gin.Context, cfg *Config, value string) { + writeCookie(c, cfg, StateCookieName, value, int(StateCookieMaxAge.Seconds()), true) +} + +func ClearStateCookie(c *gin.Context, cfg *Config) { + writeCookie(c, cfg, StateCookieName, "", -1, true) +} + +// SetCSRFCookie writes the CSRF token. Not HttpOnly so that JS can read +// it and echo the value back as the X-CSRF-Token header. +func SetCSRFCookie(c *gin.Context, cfg *Config, token string) { + writeCookie(c, cfg, CSRFCookieName, token, int(cfg.SessionTTL.Seconds()), false) +} + +func ClearCSRFCookie(c *gin.Context, cfg *Config) { + writeCookie(c, cfg, CSRFCookieName, "", -1, false) +} diff --git a/backend/helpers/oidchelper/provider.go b/backend/helpers/oidchelper/provider.go new file mode 100644 index 00000000000..bf2b10ac55b --- /dev/null +++ b/backend/helpers/oidchelper/provider.go @@ -0,0 +1,126 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "golang.org/x/oauth2" +) + +// minRefreshInterval rate-limits provider re-discovery so a flood of bad +// tokens cannot stampede the IdP's well-known endpoint. +const minRefreshInterval = 30 * time.Second + +// Provider lazily resolves the OIDC discovery document for one IdP and +// caches the resulting *oidc.Provider. Re-initializes on signature- +// verification failure to handle JWKS key rotation. +type Provider struct { + cfg *ProviderConfig + + mu sync.RWMutex + provider *oidc.Provider + lastRefresh time.Time +} + +func NewProvider(cfg *ProviderConfig) *Provider { + return &Provider{cfg: cfg} +} + +func (p *Provider) Name() string { return p.cfg.Name } + +func (p *Provider) OIDC(ctx context.Context) (*oidc.Provider, error) { + p.mu.RLock() + cached := p.provider + p.mu.RUnlock() + if cached != nil { + return cached, nil + } + return p.refresh(ctx) +} + +// refresh re-initializes the provider, but no more often than +// minRefreshInterval to prevent a stampede of failed verifications from +// hammering the IdP. Callers receive the cached provider when rate-limited. +func (p *Provider) refresh(ctx context.Context) (*oidc.Provider, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.provider != nil && time.Since(p.lastRefresh) < minRefreshInterval { + return p.provider, nil + } + prov, err := oidc.NewProvider(ctx, p.cfg.IssuerURL) + if err != nil { + return nil, fmt.Errorf("oidc discovery (%s): %w", p.cfg.IssuerURL, err) + } + p.provider = prov + p.lastRefresh = time.Now() + return prov, nil +} + +func (p *Provider) OAuth2Config(ctx context.Context) (*oauth2.Config, error) { + prov, err := p.OIDC(ctx) + if err != nil { + return nil, err + } + return &oauth2.Config{ + ClientID: p.cfg.ClientID, + ClientSecret: p.cfg.ClientSecret, + RedirectURL: p.cfg.RedirectURL, + Endpoint: prov.Endpoint(), + Scopes: p.cfg.Scopes, + }, nil +} + +// VerifyIDToken validates the raw ID token against the provider's JWKS. +// On signature-verification failure it forces a JWKS refresh (subject to +// minRefreshInterval) and retries once, transparently handling key rotation. +func (p *Provider) VerifyIDToken(ctx context.Context, raw string) (*oidc.IDToken, error) { + prov, err := p.OIDC(ctx) + if err != nil { + return nil, err + } + verifier := prov.Verifier(&oidc.Config{ClientID: p.cfg.ClientID}) + tok, err := verifier.Verify(ctx, raw) + if err == nil { + return tok, nil + } + prov, refreshErr := p.refresh(ctx) + if refreshErr != nil { + return nil, fmt.Errorf("verify id_token: %w (refresh also failed: %v)", err, refreshErr) + } + verifier = prov.Verifier(&oidc.Config{ClientID: p.cfg.ClientID}) + return verifier.Verify(ctx, raw) +} + +func (p *Provider) EndSessionURL(ctx context.Context) (string, error) { + prov, err := p.OIDC(ctx) + if err != nil { + return "", err + } + var claims struct { + EndSessionEndpoint string `json:"end_session_endpoint"` + } + if err := prov.Claims(&claims); err != nil { + return "", err + } + return claims.EndSessionEndpoint, nil +} diff --git a/backend/helpers/oidchelper/session.go b/backend/helpers/oidchelper/session.go new file mode 100644 index 00000000000..3421cb99b2a --- /dev/null +++ b/backend/helpers/oidchelper/session.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "fmt" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +const sessionIssuer = "devlake" + +// sessionParser is reused across the per-request session-validation path to +// avoid re-allocating the validator options on every call. +var sessionParser = jwt.NewParser( + jwt.WithIssuer(sessionIssuer), + jwt.WithValidMethods([]string{jwt.SigningMethodHS256.Alg()}), +) + +type SessionClaims struct { + Provider string `json:"prv,omitempty"` + Email string `json:"email,omitempty"` + Name string `json:"name,omitempty"` + jwt.RegisteredClaims +} + +// IssueSession signs a session JWT carrying the jti, the provider name (so +// /auth/logout can find the right end_session_endpoint), and the user-facing +// claims. The jti lets the server-side revocation table address one session. +func IssueSession(cfg *Config, jti, provider, sub, email, name string) (string, time.Time, error) { + now := time.Now() + expiresAt := now.Add(cfg.SessionTTL) + claims := SessionClaims{ + Provider: provider, + Email: email, + Name: name, + RegisteredClaims: jwt.RegisteredClaims{ + ID: jti, + Issuer: sessionIssuer, + Subject: sub, + IssuedAt: jwt.NewNumericDate(now), + ExpiresAt: jwt.NewNumericDate(expiresAt), + NotBefore: jwt.NewNumericDate(now), + }, + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + signed, err := token.SignedString(cfg.SessionSecret) + if err != nil { + return "", time.Time{}, err + } + return signed, expiresAt, nil +} + +func ParseSession(secret []byte, raw string) (*SessionClaims, error) { + parsed, err := sessionParser.ParseWithClaims(raw, &SessionClaims{}, func(t *jwt.Token) (any, error) { + if _, ok := t.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", t.Header["alg"]) + } + return secret, nil + }) + if err != nil { + return nil, err + } + claims, ok := parsed.Claims.(*SessionClaims) + if !ok || !parsed.Valid { + return nil, fmt.Errorf("invalid session token") + } + return claims, nil +} diff --git a/backend/helpers/oidchelper/session_test.go b/backend/helpers/oidchelper/session_test.go new file mode 100644 index 00000000000..588210e88b0 --- /dev/null +++ b/backend/helpers/oidchelper/session_test.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "testing" + "time" +) + +func newTestCfg(secret string, ttl time.Duration) *Config { + return &Config{SessionSecret: []byte(secret), SessionTTL: ttl} +} + +func TestSessionRoundTrip(t *testing.T) { + cfg := newTestCfg("session-test-secret-32-bytes!!", time.Hour) + jwt, exp, err := IssueSession(cfg, "jti-1", "entra", "user-1", "u@example.com", "Alice") + if err != nil { + t.Fatalf("issue: %v", err) + } + if exp.IsZero() { + t.Fatalf("expected non-zero expiry") + } + claims, err := ParseSession(cfg.SessionSecret, jwt) + if err != nil { + t.Fatalf("parse: %v", err) + } + if claims.Subject != "user-1" || claims.Email != "u@example.com" || claims.Name != "Alice" { + t.Fatalf("unexpected claims: %+v", claims) + } + if claims.ID != "jti-1" { + t.Fatalf("expected jti claim, got %q", claims.ID) + } + if claims.Provider != "entra" { + t.Fatalf("expected provider claim, got %q", claims.Provider) + } +} + +func TestSessionRejectsExpired(t *testing.T) { + cfg := newTestCfg("session-test-secret-32-bytes!!", -1*time.Second) + jwt, _, err := IssueSession(cfg, "jti-1", "entra", "user-1", "u@example.com", "Alice") + if err != nil { + t.Fatalf("issue: %v", err) + } + if _, err := ParseSession(cfg.SessionSecret, jwt); err == nil { + t.Fatal("expected expired token to fail validation") + } +} + +func TestSessionRejectsWrongSecret(t *testing.T) { + a := newTestCfg("session-test-secret-32-bytes!a", time.Hour) + b := newTestCfg("session-test-secret-32-bytes!b", time.Hour) + jwt, _, err := IssueSession(a, "jti-1", "entra", "user-1", "", "") + if err != nil { + t.Fatalf("issue: %v", err) + } + if _, err := ParseSession(b.SessionSecret, jwt); err == nil { + t.Fatal("expected wrong-secret to fail validation") + } +} + +func TestSessionRejectsTampered(t *testing.T) { + cfg := newTestCfg("session-test-secret-32-bytes!!", time.Hour) + jwt, _, err := IssueSession(cfg, "jti-1", "entra", "user-1", "", "") + if err != nil { + t.Fatalf("issue: %v", err) + } + tampered := jwt[:len(jwt)-2] + "AA" + if _, err := ParseSession(cfg.SessionSecret, tampered); err == nil { + t.Fatal("expected tampered token to fail validation") + } +} diff --git a/backend/helpers/oidchelper/state.go b/backend/helpers/oidchelper/state.go new file mode 100644 index 00000000000..d43b82fdf00 --- /dev/null +++ b/backend/helpers/oidchelper/state.go @@ -0,0 +1,118 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "time" +) + +// StatePayload is encrypted into the state cookie. The Nonce field is also +// echoed as the OIDC `state` query parameter so we can verify the user's +// browser is the one that initiated the flow. Provider names which IdP +// minted this flow so the callback handler picks the right token endpoint. +type StatePayload struct { + Provider string `json:"v"` + Nonce string `json:"n"` + ReturnURL string `json:"r"` + PKCEVerifier string `json:"p"` + IssuedAt time.Time `json:"t"` +} + +// EncodeState seals a StatePayload with AES-GCM (key derived from +// SESSION_SECRET via SHA-256) and returns a URL-safe base64 string. +func EncodeState(secret []byte, p *StatePayload) (string, error) { + gcm, err := newGCM(secret) + if err != nil { + return "", err + } + plaintext, err := json.Marshal(p) + if err != nil { + return "", err + } + nonce := make([]byte, gcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return "", err + } + ciphertext := gcm.Seal(nonce, nonce, plaintext, nil) + return base64.RawURLEncoding.EncodeToString(ciphertext), nil +} + +// DecodeState reverses EncodeState and rejects payloads older than StateCookieMaxAge. +func DecodeState(secret []byte, encoded string) (*StatePayload, error) { + gcm, err := newGCM(secret) + if err != nil { + return nil, err + } + raw, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("state decode: %w", err) + } + if len(raw) < gcm.NonceSize() { + return nil, fmt.Errorf("state too short") + } + nonce, ciphertext := raw[:gcm.NonceSize()], raw[gcm.NonceSize():] + plaintext, err := gcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return nil, fmt.Errorf("state decrypt: %w", err) + } + var p StatePayload + if err := json.Unmarshal(plaintext, &p); err != nil { + return nil, fmt.Errorf("state unmarshal: %w", err) + } + if time.Since(p.IssuedAt) > StateCookieMaxAge { + return nil, fmt.Errorf("state expired") + } + return &p, nil +} + +// NewNonce returns a 24-char base64 random string suitable for the OIDC +// `state` parameter (~144 bits of entropy). +func NewNonce() (string, error) { + b := make([]byte, 18) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +// NewCSRFToken returns a 256-bit random URL-safe string for double-submit CSRF. +func NewCSRFToken() (string, error) { + b := make([]byte, 32) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} + +func newGCM(secret []byte) (cipher.AEAD, error) { + // AES-256 needs a 32-byte key; SHA-256 of the secret gives one. + sum := sha256.Sum256(secret) + block, err := aes.NewCipher(sum[:]) + if err != nil { + return nil, err + } + return cipher.NewGCM(block) +} diff --git a/backend/helpers/oidchelper/state_test.go b/backend/helpers/oidchelper/state_test.go new file mode 100644 index 00000000000..3401cd372a1 --- /dev/null +++ b/backend/helpers/oidchelper/state_test.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "strings" + "testing" + "time" +) + +func TestEncodeDecodeStateRoundTrip(t *testing.T) { + secret := []byte("a-test-secret-with-at-least-32-bytes!") + in := &StatePayload{ + Nonce: "abc123", + ReturnURL: "/projects", + PKCEVerifier: "verifier", + IssuedAt: time.Now(), + } + encoded, err := EncodeState(secret, in) + if err != nil { + t.Fatalf("encode: %v", err) + } + out, err := DecodeState(secret, encoded) + if err != nil { + t.Fatalf("decode: %v", err) + } + if out.Nonce != in.Nonce || out.ReturnURL != in.ReturnURL || out.PKCEVerifier != in.PKCEVerifier { + t.Fatalf("round-trip mismatch: %+v vs %+v", in, out) + } +} + +func TestDecodeStateRejectsTamperedCiphertext(t *testing.T) { + secret := []byte("a-test-secret-with-at-least-32-bytes!") + encoded, err := EncodeState(secret, &StatePayload{Nonce: "n", IssuedAt: time.Now()}) + if err != nil { + t.Fatalf("encode: %v", err) + } + // Flip a char in the middle of the ciphertext (past the AES-GCM nonce + // prefix); flipping the very last char of base64 can be a no-op when + // the trailing bits are unused, defeating the test. + mid := len(encoded) / 2 + tampered := encoded[:mid] + flipChar(encoded[mid]) + encoded[mid+1:] + if _, err := DecodeState(secret, tampered); err == nil { + t.Fatal("expected decode to fail on tampered ciphertext") + } +} + +func TestDecodeStateRejectsWrongSecret(t *testing.T) { + encoded, err := EncodeState([]byte("first-secret-with-at-least-32-bytes!"), &StatePayload{Nonce: "n", IssuedAt: time.Now()}) + if err != nil { + t.Fatalf("encode: %v", err) + } + if _, err := DecodeState([]byte("other-secret-with-at-least-32-bytes!"), encoded); err == nil { + t.Fatal("expected decode to fail with a different secret") + } +} + +func TestDecodeStateRejectsExpiredPayload(t *testing.T) { + secret := []byte("a-test-secret-with-at-least-32-bytes!") + encoded, err := EncodeState(secret, &StatePayload{ + Nonce: "n", + IssuedAt: time.Now().Add(-30 * time.Minute), + }) + if err != nil { + t.Fatalf("encode: %v", err) + } + _, err = DecodeState(secret, encoded) + if err == nil || !strings.Contains(err.Error(), "expired") { + t.Fatalf("expected expired error, got: %v", err) + } +} + +func TestNewNonceUniqueness(t *testing.T) { + seen := map[string]struct{}{} + for i := 0; i < 50; i++ { + n, err := NewNonce() + if err != nil { + t.Fatalf("NewNonce: %v", err) + } + if len(n) < 20 { + t.Fatalf("nonce too short: %q", n) + } + if _, dup := seen[n]; dup { + t.Fatalf("duplicate nonce: %q", n) + } + seen[n] = struct{}{} + } +} + +func flipChar(b byte) string { + if b == 'A' { + return "B" + } + return "A" +} diff --git a/backend/helpers/oidchelper/wif.go b/backend/helpers/oidchelper/wif.go new file mode 100644 index 00000000000..0b03c4345b6 --- /dev/null +++ b/backend/helpers/oidchelper/wif.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "fmt" + "os" + "sync" + "time" +) + +// AzureFederatedTokenFileEnv is the env var the azure-workload-identity +// webhook sets on the pod, pointing at the projected SA token file. +const AzureFederatedTokenFileEnv = "AZURE_FEDERATED_TOKEN_FILE" + +// wifCacheTTL is the read-cache lifetime for the federated SA token. +// Kubernetes rotates SA tokens at ~80% of their TTL (1h minimum), so 10 +// minutes is well inside the safe window. +const wifCacheTTL = 10 * time.Minute + +var ( + wifMu sync.RWMutex + wifAssertion string + wifExpires time.Time + // wifReadFile is overridable from tests so they can stub the file read + // without touching the real filesystem. + wifReadFile = os.ReadFile +) + +// FederatedAssertion returns the workload-identity SA token to use as the +// `client_assertion` in an OIDC code exchange against Microsoft Entra. +// Reads the file pointed to by AZURE_FEDERATED_TOKEN_FILE and caches the +// result for wifCacheTTL. +func FederatedAssertion() (string, error) { + file := os.Getenv(AzureFederatedTokenFileEnv) + if file == "" { + return "", fmt.Errorf("%s env var not set (is the azure-workload-identity webhook installed and the pod labeled)", AzureFederatedTokenFileEnv) + } + + wifMu.RLock() + if wifExpires.After(time.Now()) { + assertion := wifAssertion + wifMu.RUnlock() + return assertion, nil + } + wifMu.RUnlock() + + wifMu.Lock() + defer wifMu.Unlock() + // Double-check under the write lock in case a peer goroutine already + // refreshed the assertion. + if wifExpires.After(time.Now()) { + return wifAssertion, nil + } + content, err := wifReadFile(file) + if err != nil { + return "", fmt.Errorf("read %s: %w", file, err) + } + wifAssertion = string(content) + wifExpires = time.Now().Add(wifCacheTTL) + return wifAssertion, nil +} diff --git a/backend/helpers/oidchelper/wif_test.go b/backend/helpers/oidchelper/wif_test.go new file mode 100644 index 00000000000..89c32614454 --- /dev/null +++ b/backend/helpers/oidchelper/wif_test.go @@ -0,0 +1,109 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oidchelper + +import ( + "errors" + "sync/atomic" + "testing" + "time" +) + +func resetFederatedAssertionCache() { + wifMu.Lock() + defer wifMu.Unlock() + wifAssertion = "" + wifExpires = time.Time{} +} + +func TestFederatedAssertionMissingEnv(t *testing.T) { + resetFederatedAssertionCache() + t.Setenv(AzureFederatedTokenFileEnv, "") + if _, err := FederatedAssertion(); err == nil { + t.Fatal("expected error when env var is unset") + } +} + +func TestFederatedAssertionReadsAndCaches(t *testing.T) { + resetFederatedAssertionCache() + var reads atomic.Int32 + wifReadFile = func(string) ([]byte, error) { + reads.Add(1) + return []byte("token-content"), nil + } + t.Cleanup(func() { wifReadFile = osReadFileForTests }) + + t.Setenv(AzureFederatedTokenFileEnv, "/fake/path") + + for i := 0; i < 5; i++ { + got, err := FederatedAssertion() + if err != nil { + t.Fatalf("call %d: %v", i, err) + } + if got != "token-content" { + t.Fatalf("got %q", got) + } + } + if got := reads.Load(); got != 1 { + t.Fatalf("expected 1 file read across 5 calls, got %d", got) + } +} + +func TestFederatedAssertionRefreshesAfterTTL(t *testing.T) { + resetFederatedAssertionCache() + var counter atomic.Int32 + wifReadFile = func(string) ([]byte, error) { + counter.Add(1) + return []byte("v"), nil + } + t.Cleanup(func() { wifReadFile = osReadFileForTests }) + + t.Setenv(AzureFederatedTokenFileEnv, "/fake/path") + + if _, err := FederatedAssertion(); err != nil { + t.Fatalf("first read: %v", err) + } + // Force the cache stale. + wifMu.Lock() + wifExpires = time.Now().Add(-time.Second) + wifMu.Unlock() + + if _, err := FederatedAssertion(); err != nil { + t.Fatalf("second read: %v", err) + } + if got := counter.Load(); got != 2 { + t.Fatalf("expected 2 file reads, got %d", got) + } +} + +func TestFederatedAssertionPropagatesReadError(t *testing.T) { + resetFederatedAssertionCache() + wifReadFile = func(string) ([]byte, error) { return nil, errors.New("boom") } + t.Cleanup(func() { wifReadFile = osReadFileForTests }) + + t.Setenv(AzureFederatedTokenFileEnv, "/fake/path") + if _, err := FederatedAssertion(); err == nil { + t.Fatal("expected error from underlying read") + } +} + +// osReadFileForTests restores the real os.ReadFile after each test that +// monkey-patches wifReadFile. Defined here to avoid an import-cycle. +var osReadFileForTests = func(name string) ([]byte, error) { + return nil, errors.New("real os.ReadFile not stubbed in this test") +} diff --git a/backend/helpers/pluginhelper/api/api_async_client.go b/backend/helpers/pluginhelper/api/api_async_client.go index 79ef16f1e90..c926c33ac24 100644 --- a/backend/helpers/pluginhelper/api/api_async_client.go +++ b/backend/helpers/pluginhelper/api/api_async_client.go @@ -122,7 +122,7 @@ func CreateAsyncApiClient( return nil, errors.Default.Wrap(err, "failed to create scheduler") } - // finally, wrap around api client with async sematic + // finally, wrap around api client with async semantic return &ApiAsyncClient{ apiClient, scheduler, diff --git a/backend/helpers/pluginhelper/api/api_client.go b/backend/helpers/pluginhelper/api/api_client.go index b0cfccf499f..7354a29db78 100644 --- a/backend/helpers/pluginhelper/api/api_client.go +++ b/backend/helpers/pluginhelper/api/api_client.go @@ -432,7 +432,7 @@ func UnmarshalResponse(res *http.Response, v interface{}) errors.Error { if err != nil { statusCode := res.StatusCode if statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden { - statusCode = http.StatusBadRequest // to avoid Basic Auth Dialog poping up + statusCode = http.StatusBadRequest // to avoid Basic Auth Dialog popping up } return errors.HttpStatus(statusCode).Wrap(err, fmt.Sprintf("error decoding response from %s: raw response: %s", res.Request.URL.String(), string(resBody))) } diff --git a/backend/helpers/pluginhelper/api/batch_save_divider.go b/backend/helpers/pluginhelper/api/batch_save_divider.go index 406d638e91d..393bbcc9bd9 100644 --- a/backend/helpers/pluginhelper/api/batch_save_divider.go +++ b/backend/helpers/pluginhelper/api/batch_save_divider.go @@ -73,10 +73,10 @@ func (d *BatchSaveDivider) ForType(rowType reflect.Type) (*BatchSave, errors.Err rowElemType := rowType.Elem() d.log.Debug("missing BatchSave for type %s", rowElemType.Name()) row := reflect.New(rowElemType).Interface() - // check if rowType had RawDataOrigin embeded + // check if rowType had RawDataOrigin embedded field, hasField := rowElemType.FieldByName("RawDataOrigin") if !hasField || field.Type != reflect.TypeOf(common.RawDataOrigin{}) { - return nil, errors.Default.New(fmt.Sprintf("type %s must have RawDataOrigin embeded", rowElemType.Name())) + return nil, errors.Default.New(fmt.Sprintf("type %s must have RawDataOrigin embedded", rowElemType.Name())) } d.batches[rowType] = batch if !d.incrementalMode { diff --git a/backend/helpers/pluginhelper/api/data_convertor_stateful.go b/backend/helpers/pluginhelper/api/data_convertor_stateful.go index 1c5292dc230..f742df83c73 100644 --- a/backend/helpers/pluginhelper/api/data_convertor_stateful.go +++ b/backend/helpers/pluginhelper/api/data_convertor_stateful.go @@ -192,7 +192,7 @@ func (converter *StatefulDataConverter[InputType]) Execute() errors.Error { if err != nil { return err } - // save the incremantal state + // save the incremental state return converter.SubtaskStateManager.Close() } diff --git a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go index 076b2c6f820..83fef06122b 100644 --- a/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go +++ b/backend/helpers/pluginhelper/api/ds_remote_api_scope_list_helper.go @@ -31,7 +31,7 @@ const ( RAS_ENTRY_TYPE_SCOPE = "scope" ) -// DsListRemoteScopes is the function type for listing remote scopes that must be implmeneted by the plugin +// DsListRemoteScopes is the function type for listing remote scopes that must be implemented by the plugin type DsListRemoteScopes[C plugin.ToolLayerApiConnection, S plugin.ToolLayerScope, P any] func( connection *C, apiClient plugin.ApiClient, groupId string, page P) (children []models.DsRemoteApiScopeListEntry[S], nextPage *P, errr errors.Error) diff --git a/backend/helpers/pluginhelper/api/graphql_async_client.go b/backend/helpers/pluginhelper/api/graphql_async_client.go index 2df25583ec1..61a1c9fa3ca 100644 --- a/backend/helpers/pluginhelper/api/graphql_async_client.go +++ b/backend/helpers/pluginhelper/api/graphql_async_client.go @@ -20,14 +20,16 @@ package api import ( "context" "fmt" + "strconv" + "sync" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/core/utils" - "sync" - "time" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) // GraphqlAsyncClient send graphql one by one @@ -47,30 +49,52 @@ type GraphqlAsyncClient struct { getRateCost func(q interface{}) int } +// defaultRateLimitConst is the generic fallback rate limit for GraphQL requests. +// It is used as the initial remaining quota when dynamic rate limit +// information is unavailable from the provider. +const defaultRateLimitConst = 1000 + // CreateAsyncGraphqlClient creates a new GraphqlAsyncClient func CreateAsyncGraphqlClient( taskCtx plugin.TaskContext, graphqlClient *graphql.Client, logger log.Logger, getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), + opts ...func(*GraphqlAsyncClient), ) (*GraphqlAsyncClient, errors.Error) { ctxWithCancel, cancel := context.WithCancel(taskCtx.GetContext()) + graphqlAsyncClient := &GraphqlAsyncClient{ ctx: ctxWithCancel, cancel: cancel, client: graphqlClient, logger: logger, rateExhaustCond: sync.NewCond(&sync.Mutex{}), - rateRemaining: 0, + rateRemaining: defaultRateLimitConst, getRateRemaining: getRateRemaining, } + // apply options + for _, opt := range opts { + opt(graphqlAsyncClient) + } + + // Env config wins over everything, only if explicitly set + if rateLimit := resolveRateLimit(taskCtx, logger); rateLimit != -1 { + logger.Info("GRAPHQL_RATE_LIMIT env override applied: %d (was %d)", rateLimit, graphqlAsyncClient.rateRemaining) + graphqlAsyncClient.rateRemaining = rateLimit + } + if getRateRemaining != nil { rateRemaining, resetAt, err := getRateRemaining(taskCtx.GetContext(), graphqlClient, logger) if err != nil { - panic(err) + graphqlAsyncClient.logger.Info("failed to fetch initial graphql rate limit, fallback to default: %v", err) + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) + } else { + graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) } - graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) + } else { + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) } // load retry/timeout from configuration @@ -115,6 +139,10 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese apiClient.rateExhaustCond.Signal() } go func() { + if apiClient.getRateRemaining == nil { + return + } + nextDuring := 3 * time.Minute if resetAt != nil && resetAt.After(time.Now()) { nextDuring = time.Until(*resetAt) @@ -126,7 +154,15 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese case <-time.After(nextDuring): newRateRemaining, newResetAt, err := apiClient.getRateRemaining(apiClient.ctx, apiClient.client, apiClient.logger) if err != nil { - panic(err) + apiClient.logger.Info("failed to update graphql rate limit, will retry next cycle: %v", err) + // Floor the reused value so Signal() always fires; prevents deadlock when + // rateRemaining is 0 and the rate-limit endpoint keeps erroring (e.g. GHE). + fallback := apiClient.rateRemaining + if fallback < defaultRateLimitConst { + fallback = defaultRateLimitConst + } + apiClient.updateRateRemaining(fallback, nil) + return } apiClient.updateRateRemaining(newRateRemaining, newResetAt) } @@ -218,3 +254,25 @@ func (apiClient *GraphqlAsyncClient) Wait() { func (apiClient *GraphqlAsyncClient) Release() { apiClient.cancel() } + +// WithFallbackRateLimit sets the initial/fallback rate limit used when +// rate limit information cannot be fetched dynamically. +// This value may be overridden later by getRateRemaining. +func WithFallbackRateLimit(limit int) func(*GraphqlAsyncClient) { + return func(c *GraphqlAsyncClient) { + if limit > 0 { + c.rateRemaining = limit + } + } +} + +// resolveRateLimit returns -1 if GRAPHQL_RATE_LIMIT is not set or invalid +func resolveRateLimit(taskCtx plugin.TaskContext, logger log.Logger) int { + if v := taskCtx.GetConfig("GRAPHQL_RATE_LIMIT"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil { + return parsed + } + logger.Warn(nil, "invalid GRAPHQL_RATE_LIMIT, using default") + } + return -1 +} diff --git a/backend/helpers/pluginhelper/api/graphql_collector.go b/backend/helpers/pluginhelper/api/graphql_collector.go index 888e6fdd4c4..28a5b2b9d86 100644 --- a/backend/helpers/pluginhelper/api/graphql_collector.go +++ b/backend/helpers/pluginhelper/api/graphql_collector.go @@ -28,7 +28,7 @@ import ( "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" plugin "github.com/apache/incubator-devlake/core/plugin" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) // CursorPager contains pagination information for a graphql request diff --git a/backend/helpers/pluginhelper/api/iterator.go b/backend/helpers/pluginhelper/api/iterator.go index 37ffd2b85cc..6f991eb9079 100644 --- a/backend/helpers/pluginhelper/api/iterator.go +++ b/backend/helpers/pluginhelper/api/iterator.go @@ -55,7 +55,7 @@ func NewBatchedDalCursorIterator(db dal.Dal, cursor dal.Rows, elemType reflect.T }, nil } -// HasNext increments the row curser. If we're at the end, it'll return false. +// HasNext increments the row cursor. If we're at the end, it'll return false. func (c *DalCursorIterator) HasNext() bool { return c.cursor.Next() } @@ -149,7 +149,7 @@ type QueueIterator struct { queue *Queue } -// HasNext increments the row curser. If we're at the end, it'll return false. +// HasNext increments the row cursor. If we're at the end, it'll return false. func (q *QueueIterator) HasNext() bool { return q.queue.GetCount() > 0 } diff --git a/backend/helpers/pluginhelper/api/model_api_helper.go b/backend/helpers/pluginhelper/api/model_api_helper.go index 3e35a7c5d4c..809192aec71 100644 --- a/backend/helpers/pluginhelper/api/model_api_helper.go +++ b/backend/helpers/pluginhelper/api/model_api_helper.go @@ -226,11 +226,11 @@ func parsePagination[P any](input *plugin.ApiResourceInput) (*P, errors.Error) { pagination := new(P) err := utils.DecodeMapStruct(input.Query, pagination, false) if err != nil { - return nil, errors.BadInput.Wrap(err, "faild to decode pagination from query string") + return nil, errors.BadInput.Wrap(err, "failed to decode pagination from query string") } err = utils.DecodeMapStruct(input.Params, pagination, false) if err != nil { - return nil, errors.BadInput.Wrap(err, "faild to decode pagination from path variables") + return nil, errors.BadInput.Wrap(err, "failed to decode pagination from path variables") } if e := vld.Struct(pagination); e != nil { return nil, errors.BadInput.Wrap(e, "invalid pagination parameters") diff --git a/backend/helpers/pluginhelper/api/subtask_state_manager.go b/backend/helpers/pluginhelper/api/subtask_state_manager.go index 7dd7d2370b0..ff1c8d7f4d5 100644 --- a/backend/helpers/pluginhelper/api/subtask_state_manager.go +++ b/backend/helpers/pluginhelper/api/subtask_state_manager.go @@ -20,6 +20,7 @@ package api import ( "fmt" "reflect" + "strings" "time" "github.com/apache/incubator-devlake/core/dal" @@ -88,6 +89,10 @@ func NewSubtaskStateManager(args *SubtaskCommonArgs) (stateManager *SubtaskState if err != nil { return } + preState, err = bootstrapStateFromCollectorStateIfNeeded(db, preState, args) + if err != nil { + return + } isIncremental, since := calculateStateManagerIncrementalMode(syncPolicy, preState, utils.ToJsonString(args.SubtaskConfig)) @@ -127,6 +132,56 @@ func loadPreviousState(db dal.Dal, plugin, subtask, params string) (*models.Subt return preState, nil } +func bootstrapStateFromCollectorStateIfNeeded(db dal.Dal, preState *models.SubtaskState, args *SubtaskCommonArgs) (*models.SubtaskState, errors.Error) { + if preState == nil || preState.PrevStartedAt != nil { + return preState, nil + } + if args == nil || args.Table == "" { + return preState, nil + } + if !db.HasTable(&models.CollectorLatestState{}) { + return preState, nil + } + + rawTable := args.GetRawDataTable() + if rawTable == "" { + return preState, nil + } + + collectorState := &models.CollectorLatestState{} + err := db.First( + collectorState, + dal.Where("raw_data_table = ? AND raw_data_params = ?", rawTable, preState.Params), + ) + if err != nil { + if db.IsErrorNotFound(err) || isStateTableNotReadyError(err) { + return preState, nil + } + return nil, errors.Default.Wrap(err, "failed to load collector state for subtask bootstrap") + } + + if collectorState.LatestSuccessStart != nil { + preState.PrevStartedAt = collectorState.LatestSuccessStart + } + if preState.TimeAfter == nil && collectorState.TimeAfter != nil { + preState.TimeAfter = collectorState.TimeAfter + } + + return preState, nil +} + +func isStateTableNotReadyError(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "_devlake_collector_latest_state") && + (strings.Contains(msg, "doesn't exist") || + strings.Contains(msg, "does not exist") || + strings.Contains(msg, "unknown table") || + strings.Contains(msg, "no such table")) +} + // calculateStateManagerIncrementalMode tries to calculate whether state manager should run in incremental mode and returns the state manager's 'since' time. func calculateStateManagerIncrementalMode(syncPolicy *models.SyncPolicy, preState *models.SubtaskState, newSubtaskConfig string) (bool, *time.Time) { if preState == nil || syncPolicy == nil { diff --git a/backend/helpers/pluginhelper/api/subtask_state_manager_test.go b/backend/helpers/pluginhelper/api/subtask_state_manager_test.go index 06eeaa43374..f4ba733ea34 100644 --- a/backend/helpers/pluginhelper/api/subtask_state_manager_test.go +++ b/backend/helpers/pluginhelper/api/subtask_state_manager_test.go @@ -30,6 +30,8 @@ import ( "github.com/stretchr/testify/mock" ) +const testGithubScopeParams = `{"ConnectionId":1,"Name":"test/repo"}` + func TestSubtaskStateManager(t *testing.T) { time0 := errors.Must1(time.Parse(time.RFC3339, "2020-01-01T00:00:00Z")) time1 := errors.Must1(time.Parse(time.RFC3339, "2021-01-01T00:00:00Z")) @@ -187,3 +189,170 @@ func TestSubtaskStateManager(t *testing.T) { }) } } + +func TestBootstrapStateFromCollectorStateIfNeeded(t *testing.T) { + latest := errors.Must1(time.Parse(time.RFC3339, "2026-04-27T18:05:31Z")) + timeAfter := errors.Must1(time.Parse(time.RFC3339, "2024-12-20T00:00:00Z")) + + t.Run("bootstrap prev_started_at and time_after from collector state", func(t *testing.T) { + mockDal := new(mockdal.Dal) + mockDal.On("HasTable", mock.Anything).Return(true).Once() + mockDal.On("First", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + dst := args.Get(0).(*models.CollectorLatestState) + dst.LatestSuccessStart = &latest + dst.TimeAfter = &timeAfter + }).Return(nil).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, err) + if assert.NotNil(t, bootstrapped.PrevStartedAt) { + assert.True(t, bootstrapped.PrevStartedAt.Equal(latest)) + } + if assert.NotNil(t, bootstrapped.TimeAfter) { + assert.True(t, bootstrapped.TimeAfter.Equal(timeAfter)) + } + mockDal.AssertExpectations(t) + }) + + t.Run("do not overwrite existing prev_started_at", func(t *testing.T) { + mockDal := new(mockdal.Dal) + + existing := errors.Must1(time.Parse(time.RFC3339, "2026-04-27T18:06:37Z")) + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + PrevStartedAt: &existing, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, err) + if assert.NotNil(t, bootstrapped.PrevStartedAt) { + assert.True(t, bootstrapped.PrevStartedAt.Equal(existing)) + } + mockDal.AssertNotCalled(t, "HasTable", mock.Anything) + mockDal.AssertNotCalled(t, "First", mock.Anything, mock.Anything) + }) + + t.Run("ignore not found collector state", func(t *testing.T) { + mockDal := new(mockdal.Dal) + notFoundErr := errors.Default.New("record not found") + mockDal.On("HasTable", mock.Anything).Return(true).Once() + mockDal.On("First", mock.Anything, mock.Anything).Return(notFoundErr).Once() + mockDal.On("IsErrorNotFound", notFoundErr).Return(true).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, err) + assert.Nil(t, bootstrapped.PrevStartedAt) + assert.Nil(t, bootstrapped.TimeAfter) + mockDal.AssertExpectations(t) + }) + + t.Run("return error when collector state query fails", func(t *testing.T) { + mockDal := new(mockdal.Dal) + dbErr := errors.Default.New("db unavailable") + mockDal.On("HasTable", mock.Anything).Return(true).Once() + mockDal.On("First", mock.Anything, mock.Anything).Return(dbErr).Once() + mockDal.On("IsErrorNotFound", dbErr).Return(false).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, bootstrapped) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "failed to load collector state for subtask bootstrap") + mockDal.AssertExpectations(t) + }) + + t.Run("ignore missing collector table errors", func(t *testing.T) { + mockDal := new(mockdal.Dal) + mockDal.On("HasTable", mock.Anything).Return(true).Once() + tableErr := errors.Default.New("Error 1146 (42S02): Table 'lake._devlake_collector_latest_state' doesn't exist") + mockDal.On("First", mock.Anything, mock.Anything).Return(tableErr).Once() + mockDal.On("IsErrorNotFound", tableErr).Return(false).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, err) + assert.NotNil(t, bootstrapped) + assert.Nil(t, bootstrapped.PrevStartedAt) + assert.Nil(t, bootstrapped.TimeAfter) + mockDal.AssertExpectations(t) + }) + + t.Run("do not ignore missing unrelated table errors", func(t *testing.T) { + mockDal := new(mockdal.Dal) + mockDal.On("HasTable", mock.Anything).Return(true).Once() + tableErr := errors.Default.New("Error 1146 (42S02): Table 'lake._tool_github_issues' doesn't exist") + mockDal.On("First", mock.Anything, mock.Anything).Return(tableErr).Once() + mockDal.On("IsErrorNotFound", tableErr).Return(false).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, bootstrapped) + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "failed to load collector state for subtask bootstrap") + mockDal.AssertExpectations(t) + }) + + t.Run("skip collector lookup when collector state table does not exist", func(t *testing.T) { + mockDal := new(mockdal.Dal) + mockDal.On("HasTable", mock.Anything).Return(false).Once() + + state := &models.SubtaskState{ + Plugin: "github", + Subtask: "Convert Jobs", + Params: testGithubScopeParams, + } + args := &SubtaskCommonArgs{Table: "github_api_jobs"} + + bootstrapped, err := bootstrapStateFromCollectorStateIfNeeded(mockDal, state, args) + assert.Nil(t, err) + assert.NotNil(t, bootstrapped) + assert.Nil(t, bootstrapped.PrevStartedAt) + assert.Nil(t, bootstrapped.TimeAfter) + mockDal.AssertNotCalled(t, "First", mock.Anything, mock.Anything) + mockDal.AssertExpectations(t) + }) +} + +func TestIsStateTableNotReadyError(t *testing.T) { + assert.False(t, isStateTableNotReadyError(nil)) + assert.True(t, isStateTableNotReadyError(errors.Default.New("Error 1146 (42S02): Table 'lake._devlake_collector_latest_state' doesn't exist"))) + assert.True(t, isStateTableNotReadyError(errors.Default.New("pq: relation \"_devlake_collector_latest_state\" does not exist"))) + assert.True(t, isStateTableNotReadyError(errors.Default.New("no such table: _devlake_collector_latest_state"))) + assert.False(t, isStateTableNotReadyError(errors.Default.New("Error 1146 (42S02): Table 'lake._tool_github_issues' doesn't exist"))) + assert.False(t, isStateTableNotReadyError(errors.Default.New("db unavailable"))) +} diff --git a/backend/helpers/pluginhelper/api/worker_scheduler.go b/backend/helpers/pluginhelper/api/worker_scheduler.go index d7cfc50961b..74d1c3f70a3 100644 --- a/backend/helpers/pluginhelper/api/worker_scheduler.go +++ b/backend/helpers/pluginhelper/api/worker_scheduler.go @@ -84,7 +84,7 @@ func NewWorkerScheduler( // It doesn't return error because it wouldn't be any when with a Blocking semantic, returned error does nothing but // causing confusion, more often, people thought it is returned by the task. // Since it is async task, the callframes would not be available for production mode, you can export Environment -// Varaible ASYNC_CF=true to enable callframes capturing when debugging. +// Variable ASYNC_CF=true to enable callframes capturing when debugging. // IMPORTANT: do NOT call SubmitBlocking inside the async task, it is likely to cause a deadlock, call // SubmitNonBlocking instead when number of tasks is relatively small. func (s *WorkerScheduler) SubmitBlocking(task func() errors.Error) { @@ -118,7 +118,7 @@ func (s *WorkerScheduler) SubmitBlocking(task func() errors.Error) { /* func (s *WorkerScheduler) gatherCallFrames() string { - cf := "set Environment Varaible ASYNC_CF=true to enable callframes capturing" + cf := "set Environment Variable ASYNC_CF=true to enable callframes capturing" if callframeEnabled { cf = utils.GatherCallFrames(1) } diff --git a/backend/helpers/pluginhelper/csv_file_test.go b/backend/helpers/pluginhelper/csv_file_test.go index b2444868819..0958ca7d0d8 100644 --- a/backend/helpers/pluginhelper/csv_file_test.go +++ b/backend/helpers/pluginhelper/csv_file_test.go @@ -36,8 +36,8 @@ func TestExampleCsvFile(t *testing.T) { defer iter.Close() for iter.HasNext() { row := iter.Fetch() - assert.Equal(t, row["name"], "foobar", "name not euqal") - assert.Equal(t, row["json"], `{"url": "https://example.com"}`, "json not euqal") + assert.Equal(t, row["name"], "foobar", "name not equal") + assert.Equal(t, row["json"], `{"url": "https://example.com"}`, "json not equal") } } diff --git a/backend/helpers/srvhelper/model_service_helper.go b/backend/helpers/srvhelper/model_service_helper.go index 9c3d05ecee7..0ca94827c52 100644 --- a/backend/helpers/srvhelper/model_service_helper.go +++ b/backend/helpers/srvhelper/model_service_helper.go @@ -89,7 +89,7 @@ func (srv *ModelSrvHelper[M]) ValidateModel(model *M) errors.Error { } // basic validator if e := srv.validator.Struct(model); e != nil { - return errors.BadInput.Wrap(e, "validation faild") + return errors.BadInput.Wrap(e, "validation failed") } return nil } diff --git a/backend/helpers/srvhelper/scope_service_helper_test.go b/backend/helpers/srvhelper/scope_service_helper_test.go index 1052b8164f4..54d7979f17c 100644 --- a/backend/helpers/srvhelper/scope_service_helper_test.go +++ b/backend/helpers/srvhelper/scope_service_helper_test.go @@ -35,7 +35,7 @@ func Test_setDefaultEntities(t *testing.T) { setDefaultEntities(sc1) assert.Equal(t, sc1.Entities, plugin.DOMAIN_TYPES) - // plugin embeded the common ScopeConfig + // plugin embedded the common ScopeConfig sc2 := &struct { common.ScopeConfig }{ diff --git a/backend/helpers/unithelper/table_info_checker.go b/backend/helpers/unithelper/table_info_checker.go index bece5b31f2e..27ba975b881 100644 --- a/backend/helpers/unithelper/table_info_checker.go +++ b/backend/helpers/unithelper/table_info_checker.go @@ -49,6 +49,11 @@ type TableInfoCheckerConfig struct { IgnoreTables []string } +type parsedPackage struct { + name string + files map[string]*ast.File +} + func NewTableInfoChecker(cfg TableInfoCheckerConfig) *TableInfoChecker { return &TableInfoChecker{ tables: make(map[string]struct{}), @@ -112,8 +117,8 @@ func (checker *TableInfoChecker) Verify() errors.Error { return errors.Default.New(sb.String()) } -func (checker *TableInfoChecker) parseDirRecursively(modelsDir string, additionalIgnorablePackages ...string) (map[string]*ast.Package, error) { - packagesMap := make(map[string]*ast.Package) +func (checker *TableInfoChecker) parseDirRecursively(modelsDir string, additionalIgnorablePackages ...string) (map[string]*parsedPackage, error) { + packagesMap := make(map[string]*parsedPackage) ignorablePackages := append(checker.ignoredPackages, additionalIgnorablePackages...) err := filepath.WalkDir(modelsDir, func(path string, d fs2.DirEntry, err error) error { if err != nil { @@ -122,7 +127,7 @@ func (checker *TableInfoChecker) parseDirRecursively(modelsDir string, additiona if !d.IsDir() { return nil } - packs, err := parser.ParseDir(token.NewFileSet(), path, nil, 0) + packs, err := parsePackagesInDir(path, 0) if err != nil { return err } @@ -143,10 +148,40 @@ func (checker *TableInfoChecker) parseDirRecursively(modelsDir string, additiona return packagesMap, nil } -func (checker *TableInfoChecker) getTableNameFuncs(pks map[string]*ast.Package) []*ast.FuncDecl { +func parsePackagesInDir(dir string, mode parser.Mode) (map[string]*parsedPackage, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + packs := make(map[string]*parsedPackage) + fileSet := token.NewFileSet() + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") { + continue + } + filename := filepath.Join(dir, entry.Name()) + file, err := parser.ParseFile(fileSet, filename, nil, mode) + if err != nil { + return nil, err + } + packageName := file.Name.Name + pack := packs[packageName] + if pack == nil { + pack = &parsedPackage{ + name: packageName, + files: make(map[string]*ast.File), + } + packs[packageName] = pack + } + pack.files[filename] = file + } + return packs, nil +} + +func (checker *TableInfoChecker) getTableNameFuncs(pks map[string]*parsedPackage) []*ast.FuncDecl { var funcs []*ast.FuncDecl for _, pack := range pks { - for _, f := range pack.Files { + for _, f := range pack.files { for _, d := range f.Decls { if fn, isFn := d.(*ast.FuncDecl); isFn && fn.Name.String() == "TableName" { funcs = append(funcs, fn) @@ -180,12 +215,12 @@ func (checker *TableInfoChecker) ensureCoverage() errors.Error { if strings.Count(path, string(os.PathSeparator)) != 0 { return fs2.SkipDir } - packs, err := parser.ParseDir(token.NewFileSet(), path, nil, parser.PackageClauseOnly) + packs, err := parsePackagesInDir(path, parser.PackageClauseOnly) if err != nil { return err } for _, pk := range packs { - if pk.Name == "main" { + if pk.name == "main" { packagesFound++ return fs2.SkipDir } diff --git a/backend/impls/context/default_basic_res.go b/backend/impls/context/default_basic_res.go index cb277a540b9..1ee4976dbea 100644 --- a/backend/impls/context/default_basic_res.go +++ b/backend/impls/context/default_basic_res.go @@ -36,7 +36,7 @@ func (c *DefaultBasicRes) GetConfigReader() config.ConfigReader { return c.cfg } -// GetConfig returns the value of the specificed name +// GetConfig returns the value of the specified name func (c *DefaultBasicRes) GetConfig(name string) string { return c.cfg.GetString(name) } diff --git a/backend/impls/context/default_exec_context.go b/backend/impls/context/default_exec_context.go index d1a2ceda00e..2d372f5b95c 100644 --- a/backend/impls/context/default_exec_context.go +++ b/backend/impls/context/default_exec_context.go @@ -96,6 +96,10 @@ func (c *defaultExecContext) IncProgress(progressType plugin.ProgressType, quant } } +func (c *defaultExecContext) GetProgress() int { + return int(atomic.LoadInt64(&c.current)) +} + func (c *defaultExecContext) fork(name string) *defaultExecContext { return newDefaultExecContext( c.ctx, diff --git a/backend/impls/dalgorm/dalgorm.go b/backend/impls/dalgorm/dalgorm.go index 3b11312a3c1..ba635355f41 100644 --- a/backend/impls/dalgorm/dalgorm.go +++ b/backend/impls/dalgorm/dalgorm.go @@ -277,7 +277,7 @@ func (d *Dalgorm) Delete(entity interface{}, clauses ...dal.Clause) errors.Error return d.convertGormError(buildTx(d.db, clauses).Delete(entity).Error) } -// UpdateColumn allows you to update mulitple records +// UpdateColumn allows you to update multiple records func (d *Dalgorm) UpdateColumn(entityOrTable interface{}, columnName string, value interface{}, clauses ...dal.Clause) errors.Error { d.unwrapDynamic(&entityOrTable, &clauses) if expr, ok := value.(dal.DalClause); ok { @@ -286,7 +286,7 @@ func (d *Dalgorm) UpdateColumn(entityOrTable interface{}, columnName string, val return d.convertGormError(buildTx(d.db, clauses).Update(columnName, value).Error) } -// UpdateColumns allows you to update multiple columns of mulitple records +// UpdateColumns allows you to update multiple columns of multiple records func (d *Dalgorm) UpdateColumns(entityOrTable interface{}, set []dal.DalSet, clauses ...dal.Clause) errors.Error { d.unwrapDynamic(&entityOrTable, &clauses) updatesSet := make(map[string]interface{}) diff --git a/backend/impls/logruslog/logger.go b/backend/impls/logruslog/logger.go index 857d75ac263..88287c9f553 100644 --- a/backend/impls/logruslog/logger.go +++ b/backend/impls/logruslog/logger.go @@ -72,11 +72,11 @@ func (l *DefaultLogger) Info(format string, a ...interface{}) { } func (l *DefaultLogger) Warn(err error, format string, a ...interface{}) { - l.Log(log.LOG_WARN, formatMessage(err, format, a...)) + l.Log(log.LOG_WARN, "%s", formatMessage(err, format, a...)) } func (l *DefaultLogger) Error(err error, format string, a ...interface{}) { - l.Log(log.LOG_ERROR, formatMessage(err, format, a...)) + l.Log(log.LOG_ERROR, "%s", formatMessage(err, format, a...)) } func (l *DefaultLogger) SetStream(config *log.LoggerStreamConfig) { diff --git a/backend/plugins/ae/api/connection.go b/backend/plugins/ae/api/connection.go index 2e42c6152ee..cc3a3889e6e 100644 --- a/backend/plugins/ae/api/connection.go +++ b/backend/plugins/ae/api/connection.go @@ -52,7 +52,7 @@ func testConnection(ctx context.Context, connection models.AeConn) (*plugin.ApiR return &plugin.ApiResourceOutput{Body: true, Status: 200}, nil case 401: // error secretKey or nonceStr return &plugin.ApiResourceOutput{Body: false, Status: http.StatusBadRequest}, nil - default: // unknow what happen , back to user + default: // unknown what happen , back to user return &plugin.ApiResourceOutput{Body: res.Body, Status: res.StatusCode}, nil } } diff --git a/backend/plugins/argocd/api/remote_api.go b/backend/plugins/argocd/api/remote_api.go index 96f27fbf302..e34e7096453 100644 --- a/backend/plugins/argocd/api/remote_api.go +++ b/backend/plugins/argocd/api/remote_api.go @@ -63,6 +63,9 @@ func listArgocdRemoteScopes( } `json:"metadata"` Spec struct { Project string `json:"project"` + Source struct { + RepoURL string `json:"repoURL"` + } `json:"source"` } `json:"spec"` } `json:"items"` } @@ -107,6 +110,9 @@ func listArgocdRemoteScopes( } `json:"metadata"` Spec struct { Project string `json:"project"` + Source struct { + RepoURL string `json:"repoURL"` + } `json:"source"` } `json:"spec"` } `json:"items"` } @@ -126,6 +132,7 @@ func listArgocdRemoteScopes( Name: item.Metadata.Name, Namespace: item.Metadata.Namespace, Project: item.Spec.Project, + RepoURL: item.Spec.Source.RepoURL, } app.ConnectionId = connection.ID diff --git a/backend/plugins/argocd/models/application.go b/backend/plugins/argocd/models/application.go index fd15a098bb1..ad7c8877081 100644 --- a/backend/plugins/argocd/models/application.go +++ b/backend/plugins/argocd/models/application.go @@ -31,7 +31,7 @@ type ArgocdApplication struct { Name string `gorm:"type:varchar(255);primaryKey" json:"name" mapstructure:"name" validate:"required"` Namespace string `gorm:"type:varchar(255)" json:"namespace" mapstructure:"namespace"` Project string `gorm:"type:varchar(255)" json:"project" mapstructure:"project"` - RepoURL string `gorm:"type:varchar(500)" json:"repoUrl" mapstructure:"repoUrl"` + RepoURL string `gorm:"type:varchar(500)" json:"repoURL" mapstructure:"repoURL"` Path string `gorm:"type:varchar(255)" json:"path" mapstructure:"path"` TargetRevision string `gorm:"type:varchar(255)" json:"targetRevision" mapstructure:"targetRevision"` DestServer string `gorm:"type:varchar(255)" json:"destServer" mapstructure:"destServer"` diff --git a/backend/plugins/argocd/models/migrationscripts/20260331_add_repo_url_to_sync_operations.go b/backend/plugins/argocd/models/migrationscripts/20260331_add_repo_url_to_sync_operations.go new file mode 100644 index 00000000000..5acdc543b9c --- /dev/null +++ b/backend/plugins/argocd/models/migrationscripts/20260331_add_repo_url_to_sync_operations.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addRepoURLToSyncOperations)(nil) + +type addRepoURLToSyncOperations struct{} + +// addRepoURLSyncOpArchived is a snapshot of ArgocdSyncOperation used solely +// for this migration so the live model can evolve independently. +type addRepoURLSyncOpArchived struct { + ConnectionId uint64 `gorm:"primaryKey"` + ApplicationName string `gorm:"primaryKey;type:varchar(255)"` + DeploymentId int64 `gorm:"primaryKey"` + RepoURL string `gorm:"type:varchar(500)"` +} + +func (addRepoURLSyncOpArchived) TableName() string { + return "_tool_argocd_sync_operations" +} + +func (m *addRepoURLToSyncOperations) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + return db.AutoMigrate(&addRepoURLSyncOpArchived{}) +} + +func (*addRepoURLToSyncOperations) Version() uint64 { + return 20260331000000 +} + +func (*addRepoURLToSyncOperations) Name() string { + return "argocd add repo_url to sync operations" +} diff --git a/backend/plugins/argocd/models/migrationscripts/register.go b/backend/plugins/argocd/models/migrationscripts/register.go index 2c70d811548..9ed9242062c 100644 --- a/backend/plugins/argocd/models/migrationscripts/register.go +++ b/backend/plugins/argocd/models/migrationscripts/register.go @@ -25,5 +25,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(addImageSupportArtifacts), + new(addRepoURLToSyncOperations), } } diff --git a/backend/plugins/argocd/models/sync_operation.go b/backend/plugins/argocd/models/sync_operation.go index 77cc095447d..ea8838cb108 100644 --- a/backend/plugins/argocd/models/sync_operation.go +++ b/backend/plugins/argocd/models/sync_operation.go @@ -28,6 +28,7 @@ type ArgocdSyncOperation struct { ApplicationName string `gorm:"primaryKey;type:varchar(255)"` DeploymentId int64 `gorm:"primaryKey"` // History ID from ArgoCD Revision string `gorm:"type:varchar(255)"` // Git SHA + RepoURL string `gorm:"type:varchar(500)"` // Git repo URL resolved from source/sources at extraction time Kind string `gorm:"type:varchar(100)"` // Kubernetes resource kind: Deployment, ReplicaSet, Rollout, StatefulSet, DaemonSet, etc. StartedAt *time.Time FinishedAt *time.Time diff --git a/backend/plugins/argocd/tasks/application_extractor.go b/backend/plugins/argocd/tasks/application_extractor.go index 3a0568609a5..1c789999c79 100644 --- a/backend/plugins/argocd/tasks/application_extractor.go +++ b/backend/plugins/argocd/tasks/application_extractor.go @@ -38,6 +38,13 @@ var ExtractApplicationsMeta = plugin.SubTaskMeta{ ProductTables: []string{models.ArgocdApplication{}.TableName()}, } +type ArgocdApiApplicationSource struct { + RepoURL string `json:"repoURL"` + Path string `json:"path"` + TargetRevision string `json:"targetRevision"` + Chart string `json:"chart"` +} + type ArgocdApiApplication struct { Metadata struct { Name string `json:"name"` @@ -46,11 +53,9 @@ type ArgocdApiApplication struct { } `json:"metadata"` Spec struct { Project string `json:"project"` - Source struct { - RepoURL string `json:"repoURL"` - Path string `json:"path"` - TargetRevision string `json:"targetRevision"` - } `json:"source"` + // Single-source apps use Source; multi-source apps use Sources. + Source ArgocdApiApplicationSource `json:"source"` + Sources []ArgocdApiApplicationSource `json:"sources"` Destination struct { Server string `json:"server"` Namespace string `json:"namespace"` @@ -88,13 +93,31 @@ func ExtractApplications(taskCtx plugin.SubTaskContext) errors.Error { return nil, errors.Default.Wrap(err, "error unmarshaling application") } + // Resolve the primary source. Multi-source apps populate spec.sources[] + // instead of spec.source; we prefer the first git-hosted source so that + // cicd_deployment_commits.repo_url is a browsable repository URL rather + // than a Helm chart registry address. + primarySource := apiApp.Spec.Source + if primarySource.RepoURL == "" && len(apiApp.Spec.Sources) > 0 { + for _, src := range apiApp.Spec.Sources { + if isGitHostedURL(src.RepoURL) { + primarySource = src + break + } + } + // Fallback: use the first source if none matched the git-host heuristic. + if primarySource.RepoURL == "" { + primarySource = apiApp.Spec.Sources[0] + } + } + application := &models.ArgocdApplication{ Name: apiApp.Metadata.Name, Namespace: apiApp.Metadata.Namespace, Project: apiApp.Spec.Project, - RepoURL: apiApp.Spec.Source.RepoURL, - Path: apiApp.Spec.Source.Path, - TargetRevision: apiApp.Spec.Source.TargetRevision, + RepoURL: primarySource.RepoURL, + Path: primarySource.Path, + TargetRevision: primarySource.TargetRevision, DestServer: apiApp.Spec.Destination.Server, DestNamespace: apiApp.Spec.Destination.Namespace, SyncStatus: apiApp.Status.Sync.Status, diff --git a/backend/plugins/argocd/tasks/sync_operation_collector.go b/backend/plugins/argocd/tasks/sync_operation_collector.go index 19d94a98d2a..6f58a16e2a2 100644 --- a/backend/plugins/argocd/tasks/sync_operation_collector.go +++ b/backend/plugins/argocd/tasks/sync_operation_collector.go @@ -38,6 +38,7 @@ var CollectSyncOperationsMeta = plugin.SubTaskMeta{ EntryPoint: CollectSyncOperations, EnabledByDefault: true, Description: "Collect sync operations (deployment history) from ArgoCD API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, DependencyTables: []string{models.ArgocdApplication{}.TableName()}, } diff --git a/backend/plugins/argocd/tasks/sync_operation_convertor.go b/backend/plugins/argocd/tasks/sync_operation_convertor.go index fc748ae55f5..e577b7af185 100644 --- a/backend/plugins/argocd/tasks/sync_operation_convertor.go +++ b/backend/plugins/argocd/tasks/sync_operation_convertor.go @@ -40,6 +40,7 @@ var ConvertSyncOperationsMeta = plugin.SubTaskMeta{ EntryPoint: ConvertSyncOperations, EnabledByDefault: true, Description: "Convert sync operations to domain layer deployments", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, DependencyTables: []string{models.ArgocdSyncOperation{}.TableName()}, ProductTables: []string{"cicd_deployments", "cicd_deployment_commits"}, } @@ -136,8 +137,14 @@ func ConvertSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { results = append(results, deployment) if syncOp.Revision != "" { + // Priority: repo_url resolved at extraction time (always present for + // multi-source apps) → application-level repo_url → deployment name + // as a last-resort non-empty placeholder. repoUrl := deployment.Name - if application != nil && application.RepoURL != "" { + switch { + case syncOp.RepoURL != "": + repoUrl = syncOp.RepoURL + case application != nil && application.RepoURL != "": repoUrl = application.RepoURL } diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor.go b/backend/plugins/argocd/tasks/sync_operation_extractor.go index 5de738a38cc..6ea7f785891 100644 --- a/backend/plugins/argocd/tasks/sync_operation_extractor.go +++ b/backend/plugins/argocd/tasks/sync_operation_extractor.go @@ -38,19 +38,28 @@ var ExtractSyncOperationsMeta = plugin.SubTaskMeta{ EntryPoint: ExtractSyncOperations, EnabledByDefault: true, Description: "Extract sync operations from raw data", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, DependencyTables: []string{RAW_SYNC_OPERATION_TABLE}, ProductTables: []string{models.ArgocdSyncOperation{}.TableName()}, } +// ArgocdApiSyncSource represents a single source in a multi-source ArgoCD application. +type ArgocdApiSyncSource struct { + RepoURL string `json:"repoURL"` + Chart string `json:"chart"` +} + type ArgocdApiSyncOperation struct { // For history entries ID int64 `json:"id"` Revision string `json:"revision"` + Revisions []string `json:"revisions"` // multi-source apps populate this instead of revision DeployedAt time.Time `json:"deployedAt"` DeployStartedAt *time.Time `json:"deployStartedAt"` Source struct { RepoURL string `json:"repoURL"` } `json:"source"` + Sources []ArgocdApiSyncSource `json:"sources"` // multi-source apps populate this instead of source InitiatedBy struct { Username string `json:"username"` Automated bool `json:"automated"` @@ -65,6 +74,7 @@ type ArgocdApiSyncOperation struct { FinishedAt *time.Time `json:"finishedAt"` SyncResult struct { Revision string `json:"revision"` + Revisions []string `json:"revisions"` // multi-source apps Resources []ArgocdApiSyncResourceItem `json:"resources"` } `json:"syncResult"` } @@ -178,10 +188,27 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { isOperationState := apiOp.Phase != "" + // operationState always represents the same sync as the latest history entry + // (when phase=Succeeded). Skip it to avoid the duplicate DeploymentId. + if isOperationState && isRedundantOperationState(apiOp.Phase) { + return nil, nil + } + + // For multi-source apps ArgoCD sets revisions[] instead of revision. Resolve + // the single commit SHA we care about before deciding whether to skip this entry. + if apiOp.Revision == "" { + apiOp.Revision = resolveMultiSourceRevision(apiOp.Revisions, apiOp.Sources) + } if !isOperationState && apiOp.DeployedAt.IsZero() && apiOp.Revision == "" { return nil, nil } + // Resolve the git repo URL at extraction time so the convertor can set + // cicd_deployment_commits.repo_url correctly even when + // _tool_argocd_applications.repo_url is empty (e.g. for multi-source apps + // whose collectApplications subtask was skipped due to state caching). + syncOp.RepoURL = resolveGitRepoURL(apiOp.Source.RepoURL, apiOp.Sources) + if isOperationState { start := normalize(apiOp.StartedAt) if start != nil { @@ -189,7 +216,14 @@ func ExtractSyncOperations(taskCtx plugin.SubTaskContext) errors.Error { } else { syncOp.DeploymentId = time.Now().Unix() } - syncOp.Revision = apiOp.SyncResult.Revision + // Prefer the top-level resolved revision; fall back to syncResult. + syncOp.Revision = apiOp.Revision + if syncOp.Revision == "" { + syncOp.Revision = resolveMultiSourceRevision(apiOp.SyncResult.Revisions, apiOp.Sources) + } + if syncOp.Revision == "" { + syncOp.Revision = apiOp.SyncResult.Revision + } syncOp.StartedAt = start syncOp.FinishedAt = normalizePtr(apiOp.FinishedAt) syncOp.Phase = apiOp.Phase @@ -379,3 +413,148 @@ func stringSlicesEqual(a, b []string) bool { } return true } + +// resolveMultiSourceRevision picks the git commit SHA from a multi-source ArgoCD +// application's revisions slice. ArgoCD multi-source apps store one revision per +// source: Helm chart sources carry a semver tag while git sources carry a 40-hex +// commit SHA. We prefer the first git-hosted source (github.com / gitlab.com / +// bitbucket.org) and fall back to any entry that looks like a 40-character hex SHA. +// +// Single-source apps already populate the top-level "revision" field, so this +// function is only called when that field is empty. +func resolveMultiSourceRevision(revisions []string, sources []ArgocdApiSyncSource) string { + if len(revisions) == 0 { + return "" + } + + // Pass 1: prefer a revision whose corresponding source is a git hosting service. + for i, rev := range revisions { + if i >= len(sources) { + break + } + repoURL := sources[i].RepoURL + if isGitHostedURL(repoURL) && isCommitSHA(rev) { + return rev + } + } + + // Pass 2: accept any revision that looks like a full commit SHA regardless of + // source type (covers self-hosted Gitea / Forgejo / etc.). + for _, rev := range revisions { + if isCommitSHA(rev) { + return rev + } + } + + return "" +} + +// isGitHostedURL returns true when the URL belongs to a known git hosting service +// or is clearly not a Helm chart registry. +func isGitHostedURL(repoURL string) bool { + if repoURL == "" { + return false + } + gitHosts := []string{ + "github.com", + "gitlab.com", + "bitbucket.org", + "dev.azure.com", + "ssh.dev.azure.com", + "gitea.", + "forgejo.", + } + lower := strings.ToLower(repoURL) + for _, host := range gitHosts { + if strings.Contains(lower, host) { + return true + } + } + // Any https/ssh git URL that is not a chart registry (gs://, oci://, https://*.azurecr.io, etc.) + chartPrefixes := []string{"gs://", "oci://", "s3://"} + for _, pfx := range chartPrefixes { + if strings.HasPrefix(lower, pfx) { + return false + } + } + // .git suffix is a strong signal + return strings.HasSuffix(strings.TrimSpace(repoURL), ".git") +} + +// resolveGitRepoURL returns the best git repository URL from a sync operation's +// source metadata. For single-source apps the source.repoURL is used directly. +// For multi-source apps (sources[]) the first URL that matches a known git +// hosting service is preferred; if none match the heuristic, the first non-chart +// HTTPS/SSH URL is used as a fallback so that cicd_deployment_commits.repo_url +// is never left as the deployment-name placeholder. +// +// This is called during extractSyncOperations which always runs, providing +// reliable repo_url population even when extractApplications is skipped due +// to the collector state cache. +func resolveGitRepoURL(singleSourceURL string, sources []ArgocdApiSyncSource) string { + // Single-source app: use the URL directly. + if singleSourceURL != "" { + return singleSourceURL + } + + // Multi-source app: pass 1 — prefer a known git host. + for _, src := range sources { + if isGitHostedURL(src.RepoURL) { + return src.RepoURL + } + } + + // Pass 2 — fall back to the first non-chart URL (covers self-hosted instances + // not in the known-host list, e.g. on-prem GitLab with a custom domain). + chartPrefixes := []string{"gs://", "oci://", "s3://"} + for _, src := range sources { + if src.RepoURL == "" { + continue + } + lower := strings.ToLower(src.RepoURL) + isChart := false + for _, pfx := range chartPrefixes { + if strings.HasPrefix(lower, pfx) { + isChart = true + break + } + } + if !isChart { + return src.RepoURL + } + } + + return "" +} + +// isCommitSHA returns true for a 40-character lowercase hexadecimal string, +// which is the standard representation of a Git commit SHA-1. +func isCommitSHA(s string) bool { + if len(s) != 40 { + return false + } + for _, c := range s { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + return true +} + +// isRedundantOperationState reports whether an operationState entry should be +// skipped because it duplicates the most recent history entry. +// +// ArgoCD exposes a completed sync in two places: +// - status.history[] — authoritative log, one entry per sync +// - status.operationState — mirrors the last sync while it is current +// +// When phase is Succeeded the operationState is always already present in +// history[]. Emitting it would produce a second ArgocdSyncOperation whose +// DeploymentId differs by one second (startedAt vs deployedAt), creating a +// duplicate deployment in DORA metrics. +// +// In-flight (Running, Terminating) and terminal-failure (Failed, Error) states +// are not yet — or may never be — written to history, so they must be kept. +func isRedundantOperationState(phase string) bool { + return phase == "Succeeded" +} diff --git a/backend/plugins/argocd/tasks/sync_operation_extractor_test.go b/backend/plugins/argocd/tasks/sync_operation_extractor_test.go index 9d71cec41af..f3f9e23ca18 100644 --- a/backend/plugins/argocd/tasks/sync_operation_extractor_test.go +++ b/backend/plugins/argocd/tasks/sync_operation_extractor_test.go @@ -76,3 +76,171 @@ func TestCollectContainerImages_FallbackRevisionAndSummary(t *testing.T) { // normalizeImages: dedupe + sort assert.Equal(t, []string{"a", "b"}, normalizeImages([]string{"b", "a", "b"})) } + +// ── resolveMultiSourceRevision ──────────────────────────────────────────────── + +func TestResolveMultiSourceRevision_GitHubSourceWins(t *testing.T) { + // Multi-source pattern: Helm chart (GCS) + git values repo (GitHub). + revisions := []string{"2.6.2", "5dd95b4efd7e9b668c361bbddb8d7f1e56c32ac1"} + sources := []ArgocdApiSyncSource{ + {RepoURL: "gs://charts-example-net/infra/stable", Chart: "generic-service"}, + {RepoURL: "https://github.com/example/my-repo"}, + } + got := resolveMultiSourceRevision(revisions, sources) + assert.Equal(t, "5dd95b4efd7e9b668c361bbddb8d7f1e56c32ac1", got) +} + +func TestResolveMultiSourceRevision_GitLabSourceWins(t *testing.T) { + revisions := []string{"1.0.0", "aabbccdd11223344aabbccdd11223344aabbccdd"} + sources := []ArgocdApiSyncSource{ + {RepoURL: "oci://registry.example.com/charts", Chart: "app"}, + {RepoURL: "https://gitlab.com/example/config"}, + } + got := resolveMultiSourceRevision(revisions, sources) + assert.Equal(t, "aabbccdd11223344aabbccdd11223344aabbccdd", got) +} + +func TestResolveMultiSourceRevision_FallbackToAnySHA(t *testing.T) { + // Neither source matches a known git hosting service (no github/gitlab/gitea/etc. + // prefix). The function should still return the 40-hex SHA via the fallback + // pass that accepts any commit-SHA-shaped revision regardless of source type. + revisions := []string{"1.2.3", "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"} + sources := []ArgocdApiSyncSource{ + {RepoURL: "gs://bucket/charts"}, + {RepoURL: "https://git.acme-corp.internal/team/config"}, + } + got := resolveMultiSourceRevision(revisions, sources) + assert.Equal(t, "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", got) +} + +func TestResolveMultiSourceRevision_EmptyRevisions(t *testing.T) { + assert.Equal(t, "", resolveMultiSourceRevision(nil, nil)) + assert.Equal(t, "", resolveMultiSourceRevision([]string{}, []ArgocdApiSyncSource{})) +} + +func TestResolveMultiSourceRevision_AllSemver(t *testing.T) { + // All revisions are semver tags; nothing looks like a commit SHA. + revisions := []string{"1.0.0", "2.3.4"} + sources := []ArgocdApiSyncSource{ + {RepoURL: "oci://registry.example.com/charts"}, + {RepoURL: "oci://registry.example.com/other"}, + } + assert.Equal(t, "", resolveMultiSourceRevision(revisions, sources)) +} + +func TestResolveMultiSourceRevision_SingleGitSHA(t *testing.T) { + // Single-source multi-source edge case. + revisions := []string{"abcdef1234567890abcdef1234567890abcdef12"} + sources := []ArgocdApiSyncSource{{RepoURL: "https://github.com/example/repo"}} + got := resolveMultiSourceRevision(revisions, sources) + assert.Equal(t, "abcdef1234567890abcdef1234567890abcdef12", got) +} + +// ── isCommitSHA ─────────────────────────────────────────────────────────────── + +func TestIsCommitSHA(t *testing.T) { + assert.True(t, isCommitSHA("5dd95b4efd7e9b668c361bbddb8d7f1e56c32ac1")) + assert.True(t, isCommitSHA("AABBCCDD11223344AABBCCDD11223344AABBCCDD")) + assert.False(t, isCommitSHA("2.6.2")) + assert.False(t, isCommitSHA("")) + assert.False(t, isCommitSHA("5dd95b4efd7e9b668c361bbddb8d7f1e56c32ac")) // 39 chars + assert.False(t, isCommitSHA("5dd95b4efd7e9b668c361bbddb8d7f1e56c32ac12")) // 41 chars +} + +// ── resolveGitRepoURL ───────────────────────────────────────────────────────── + +func TestResolveGitRepoURL_SingleSource(t *testing.T) { + // Single-source app: singleSourceURL is used directly, sources ignored. + got := resolveGitRepoURL("https://github.com/example/my-app", nil) + assert.Equal(t, "https://github.com/example/my-app", got) +} + +func TestResolveGitRepoURL_MultiSourceGitHubWins(t *testing.T) { + // Multi-source pattern: GCS chart + GitHub values ref. + sources := []ArgocdApiSyncSource{ + {RepoURL: "gs://charts-example-net/infra/stable", Chart: "generic-service"}, + {RepoURL: "https://github.com/example/my-app"}, + } + got := resolveGitRepoURL("", sources) + assert.Equal(t, "https://github.com/example/my-app", got) +} + +func TestResolveGitRepoURL_MultiSourceOCIChart(t *testing.T) { + // OCI chart + GitLab values repo. + sources := []ArgocdApiSyncSource{ + {RepoURL: "oci://registry.example.com/charts", Chart: "app"}, + {RepoURL: "https://gitlab.com/org/config"}, + } + got := resolveGitRepoURL("", sources) + assert.Equal(t, "https://gitlab.com/org/config", got) +} + +func TestResolveGitRepoURL_FallbackNonChartURL(t *testing.T) { + // No known git host but a non-chart HTTPS URL is still better than nothing. + sources := []ArgocdApiSyncSource{ + {RepoURL: "gs://bucket/charts"}, + {RepoURL: "https://git.acme-corp.internal/team/config"}, + } + got := resolveGitRepoURL("", sources) + assert.Equal(t, "https://git.acme-corp.internal/team/config", got) +} + +func TestResolveGitRepoURL_AllChartSources(t *testing.T) { + // All sources are chart registries — returns empty string. + sources := []ArgocdApiSyncSource{ + {RepoURL: "gs://charts-example-net/infra/stable"}, + {RepoURL: "oci://registry.example.com/charts"}, + } + got := resolveGitRepoURL("", sources) + assert.Equal(t, "", got) +} + +func TestResolveGitRepoURL_EmptySources(t *testing.T) { + assert.Equal(t, "", resolveGitRepoURL("", nil)) + assert.Equal(t, "", resolveGitRepoURL("", []ArgocdApiSyncSource{})) +} + +// ── isGitHostedURL ──────────────────────────────────────────────────────────── + +func TestIsGitHostedURL(t *testing.T) { + assert.True(t, isGitHostedURL("https://github.com/org/repo")) + assert.True(t, isGitHostedURL("git@github.com:org/repo.git")) + assert.True(t, isGitHostedURL("https://gitlab.com/org/repo")) + assert.True(t, isGitHostedURL("https://bitbucket.org/org/repo")) + assert.True(t, isGitHostedURL("https://dev.azure.com/org/proj/_git/repo")) + assert.True(t, isGitHostedURL("https://gitea.internal.corp/team/config")) + assert.True(t, isGitHostedURL("https://example.com/repo.git")) + + assert.False(t, isGitHostedURL("gs://charts-example-net/infra/stable")) + assert.False(t, isGitHostedURL("oci://registry.example.com/charts")) + assert.False(t, isGitHostedURL("s3://my-bucket/charts")) + assert.False(t, isGitHostedURL("")) +} + +// ── isRedundantOperationState ───────────────────────────────────────────────── + +func TestIsRedundantOperationState_SucceededIsSkipped(t *testing.T) { + assert.True(t, isRedundantOperationState("Succeeded"), + "Succeeded operationState always duplicates the latest history entry") +} + +func TestIsRedundantOperationState_NonTerminalPhasesAreKept(t *testing.T) { + for _, phase := range []string{"Running", "Terminating"} { + assert.False(t, isRedundantOperationState(phase), + "in-flight phase %q is not yet in history and must be kept", phase) + } +} + +func TestIsRedundantOperationState_FailurePhasesAreKept(t *testing.T) { + for _, phase := range []string{"Failed", "Error"} { + assert.False(t, isRedundantOperationState(phase), + "failure phase %q may never reach history and must be kept", phase) + } +} + +func TestIsRedundantOperationState_HistoryEntryHasNoPhase(t *testing.T) { + // History entries have phase="" — isOperationState is false before this + // function is called, but double-check the function itself is safe with + // an empty string so the guard can never accidentally suppress history rows. + assert.False(t, isRedundantOperationState("")) +} diff --git a/backend/plugins/asana/api/blueprint_v200.go b/backend/plugins/asana/api/blueprint_v200.go new file mode 100644 index 00000000000..2820593e350 --- /dev/null +++ b/backend/plugins/asana/api/blueprint_v200.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig], + connection *models.AsanaConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "asana", + subtaskMetas, + scopeConfig.Entities, + tasks.AsanaOptions{ + ConnectionId: connection.ID, + ProjectId: scope.Gid, + ScopeConfigId: scopeConfig.ID, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig], + connection *models.AsanaConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.Gid) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/asana/api/connection_api.go b/backend/plugins/asana/api/connection_api.go new file mode 100644 index 00000000000..6d69c8c4ee4 --- /dev/null +++ b/backend/plugins/asana/api/connection_api.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type AsanaTestConnResponse struct { + shared.ApiBody + Connection *models.AsanaConn +} + +func testConnection(ctx context.Context, connection models.AsanaConn) (*AsanaTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.GetEndpoint() == "" { + connection.Endpoint = defaultAsanaEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + res, err := apiClient.Get("users/me", nil, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized { + return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error while testing connection") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := AsanaTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.AsanaConn + err := helper.Decode(input.Body, &connection, vld) + if err != nil { + return nil, err + } + if connection.Endpoint == "" { + connection.Endpoint = defaultAsanaEndpoint + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + if connection.Endpoint == "" { + connection.Endpoint = defaultAsanaEndpoint + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.AsanaConn) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return &plugin.ApiResourceOutput{Body: testConnectionResult, Status: http.StatusOK}, nil +} + +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + if input.Body != nil { + if endpoint, ok := input.Body["endpoint"]; !ok || endpoint == nil || endpoint == "" { + input.Body["endpoint"] = defaultAsanaEndpoint + } + } + return dsHelper.ConnApi.Post(input) +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/asana/api/init.go b/backend/plugins/asana/api/init.go new file mode 100644 index 00000000000..d13865b184c --- /dev/null +++ b/backend/plugins/asana/api/init.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/go-playground/validator/v10" +) + +const defaultAsanaEndpoint = "https://app.asana.com/api/1.0/" + +var vld *validator.Validate +var basicRes context.BasicRes + +var dsHelper *api.DsHelper[models.AsanaConnection, models.AsanaProject, models.AsanaScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.AsanaConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.AsanaConnection, models.AsanaProject, AsanaRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.AsanaConnection, models.AsanaProject, models.AsanaScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.AsanaConnection) models.AsanaConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.AsanaConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.AsanaConnection, models.AsanaProject, AsanaRemotePagination](raProxy, listAsanaRemoteScopes) +} diff --git a/backend/plugins/asana/api/remote_api.go b/backend/plugins/asana/api/remote_api.go new file mode 100644 index 00000000000..0245cb6008b --- /dev/null +++ b/backend/plugins/asana/api/remote_api.go @@ -0,0 +1,632 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/url" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +type AsanaRemotePagination struct { + Offset string `json:"offset"` + Limit int `json:"limit"` +} + +// Response types for Asana API +type asanaWorkspaceResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + IsOrganization bool `json:"is_organization"` +} + +type asanaWorkspacesListResponse struct { + Data []asanaWorkspaceResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaNextPage struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` +} + +type asanaTeamResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Description string `json:"description"` + PermalinkUrl string `json:"permalink_url"` +} + +type asanaTeamsListResponse struct { + Data []asanaTeamResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaPortfolioResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + PermalinkUrl string `json:"permalink_url"` +} + +type asanaPortfoliosListResponse struct { + Data []asanaPortfolioResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaGoalResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Notes string `json:"notes"` +} + +type asanaGoalsListResponse struct { + Data []asanaGoalResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +type asanaProjectResponse struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Archived bool `json:"archived"` + PermalinkUrl string `json:"permalink_url"` + Workspace *struct { + Gid string `json:"gid"` + } `json:"workspace"` + Team *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"team"` +} + +type asanaProjectsListResponse struct { + Data []asanaProjectResponse `json:"data"` + NextPage *asanaNextPage `json:"next_page"` +} + +// Scope type constants +const ( + ScopeTypeTeam = "team" + ScopeTypePortfolio = "portfolio" + ScopeTypeGoal = "goal" +) + +func listAsanaRemoteScopes( + connection *models.AsanaConnection, + apiClient plugin.ApiClient, + groupId string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + if page.Limit == 0 { + page.Limit = 100 + } + + // Level 1: No groupId - list workspaces + if groupId == "" { + return listAsanaWorkspaces(apiClient, page) + } + + // Parse hierarchical groupId + // Format examples: + // - "workspace/{gid}" -> show Teams, Portfolios, Goals options + // - "workspace/{gid}/team/{teamGid}" -> list projects in team + // - "workspace/{gid}/portfolio/{portfolioGid}" -> list projects in portfolio + // - "workspace/{gid}/goal/{goalGid}" -> list projects for goal + + if strings.HasPrefix(groupId, "workspace/") { + parts := strings.Split(groupId[10:], "/") // Remove "workspace/" prefix + + if len(parts) == 1 { + // Level 2: Workspace selected - show Teams, Portfolios, Goals as categories + workspaceGid := parts[0] + return listAsanaScopeCategories(apiClient, workspaceGid, page) + } + + if len(parts) >= 3 { + workspaceGid := parts[0] + scopeType := parts[1] + scopeGid := parts[2] + + switch scopeType { + case ScopeTypeTeam: + // Level 4: List projects in team + return listAsanaTeamProjects(apiClient, workspaceGid, scopeGid, page) + case ScopeTypePortfolio: + // Level 4: List projects in portfolio + return listAsanaPortfolioProjects(apiClient, workspaceGid, scopeGid, page) + case ScopeTypeGoal: + // Level 4: List projects for goal + return listAsanaGoalProjects(apiClient, workspaceGid, scopeGid, page) + } + } + + if len(parts) == 2 { + workspaceGid := parts[0] + scopeType := parts[1] + + switch scopeType { + case ScopeTypeTeam: + // Level 3: List all teams + return listAsanaTeams(apiClient, workspaceGid, page) + case ScopeTypePortfolio: + // Level 3: List all portfolios + return listAsanaPortfolios(apiClient, workspaceGid, page) + case ScopeTypeGoal: + // Level 3: List all goals + return listAsanaGoals(apiClient, workspaceGid, page) + } + } + } + + return nil, nil, errors.BadInput.New("invalid groupId format") +} + +// Level 1: List workspaces +func listAsanaWorkspaces( + apiClient plugin.ApiClient, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,is_organization") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + res, err := apiClient.Get("workspaces", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch workspaces from Asana API") + } + + var response asanaWorkspacesListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana workspaces response") + } + + for _, workspace := range response.Data { + groupId := fmt.Sprintf("workspace/%s", workspace.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: nil, // Root level, no parent + Id: groupId, + Name: workspace.Name, + FullName: workspace.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 2: Show scope categories (Teams, Portfolios, Goals) +func listAsanaScopeCategories( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the workspace + parentId := fmt.Sprintf("workspace/%s", workspaceGid) + + // Return the three main categories: Teams, Portfolios, Goals + children = []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeTeam), + Name: "🏢 Teams", + FullName: "Teams", + }, + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypePortfolio), + Name: "📁 Portfolios", + FullName: "Portfolios", + }, + { + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeGoal), + Name: "🎯 Goals", + FullName: "Goals", + }, + } + + return children, nil, nil +} + +// Level 3: List teams in workspace +func listAsanaTeams( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the "Teams" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeTeam) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,description,permalink_url") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("workspaces/%s/teams", workspaceGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch teams from Asana API") + } + + var response asanaTeamsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana teams response") + } + + for _, team := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeTeam, team.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: team.Name, + FullName: team.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 3: List portfolios in workspace +func listAsanaPortfolios( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the "Portfolios" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypePortfolio) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("workspace", workspaceGid) + query.Set("owner", "me") + query.Set("opt_fields", "name,resource_type,permalink_url") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + res, err := apiClient.Get("portfolios", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch portfolios from Asana API") + } + + var response asanaPortfoliosListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana portfolios response") + } + + for _, portfolio := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypePortfolio, portfolio.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: portfolio.Name, + FullName: portfolio.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 3: List goals in workspace +func listAsanaGoals( + apiClient plugin.ApiClient, + workspaceGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the "Goals" category + parentId := fmt.Sprintf("workspace/%s/%s", workspaceGid, ScopeTypeGoal) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("workspace", workspaceGid) + query.Set("opt_fields", "name,resource_type,notes") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + res, err := apiClient.Get("goals", query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch goals from Asana API") + } + + var response asanaGoalsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana goals response") + } + + for _, goal := range response.Data { + groupId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeGoal, goal.Gid) + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parentId, + Id: groupId, + Name: goal.Name, + FullName: goal.Name, + }) + } + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects in a team +func listAsanaTeamProjects( + apiClient plugin.ApiClient, + workspaceGid string, + teamGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific team + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeTeam, teamGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("teams/%s/projects", teamGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch team projects from Asana API") + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana team projects response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects in a portfolio +func listAsanaPortfolioProjects( + apiClient plugin.ApiClient, + workspaceGid string, + portfolioGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific portfolio + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypePortfolio, portfolioGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + apiPath := fmt.Sprintf("portfolios/%s/items", portfolioGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to fetch portfolio items from Asana API") + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana portfolio items response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Level 4: List projects associated with a goal +func listAsanaGoalProjects( + apiClient plugin.ApiClient, + workspaceGid string, + goalGid string, + page AsanaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject], + nextPage *AsanaRemotePagination, + err errors.Error, +) { + // Parent is the specific goal + parentId := fmt.Sprintf("workspace/%s/%s/%s", workspaceGid, ScopeTypeGoal, goalGid) + + query := url.Values{} + query.Set("limit", fmt.Sprintf("%d", page.Limit)) + query.Set("opt_fields", "name,resource_type,archived,permalink_url,workspace,team") + if page.Offset != "" { + query.Set("offset", page.Offset) + } + + // Goals API: GET /goals/{goal_gid}/parentGoals for related projects + // Note: Asana's goal-to-project relationship is through supporting work + apiPath := fmt.Sprintf("goals/%s/supportingWork", goalGid) + res, err := apiClient.Get(apiPath, query, nil) + if err != nil { + // If supporting work API fails, return empty list + return children, nil, nil + } + + var response asanaProjectsListResponse + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Asana goal supporting work response") + } + + children = convertProjectsToScopes(response.Data, workspaceGid, &parentId) + + if response.NextPage != nil && response.NextPage.Offset != "" { + nextPage = &AsanaRemotePagination{ + Offset: response.NextPage.Offset, + Limit: page.Limit, + } + } + + return children, nextPage, nil +} + +// Helper function to convert Asana projects to scope entries +func convertProjectsToScopes( + projects []asanaProjectResponse, + workspaceGid string, + parentId *string, +) []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject] { + var children []dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject] + + for _, project := range projects { + workspaceGidVal := workspaceGid + if project.Workspace != nil { + workspaceGidVal = project.Workspace.Gid + } + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.AsanaProject]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: parentId, + Id: project.Gid, + Name: project.Name, + FullName: project.Name, + Data: &models.AsanaProject{ + Gid: project.Gid, + Name: project.Name, + ResourceType: project.ResourceType, + Archived: project.Archived, + PermalinkUrl: project.PermalinkUrl, + WorkspaceGid: workspaceGidVal, + }, + }) + } + + return children +} + +// RemoteScopes list all available scopes (projects) for this connection +// @Summary list all available scopes (projects) for this connection +// @Description list all available scopes (projects) for this connection +// @Tags plugins/asana +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} RemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/asana/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/asana/api/scope_api.go b/backend/plugins/asana/api/scope_api.go new file mode 100644 index 00000000000..afbc8486583 --- /dev/null +++ b/backend/plugins/asana/api/scope_api.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.AsanaProject] +type ScopeDetail api.ScopeDetail[models.AsanaProject, models.AsanaScopeConfig] + +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/asana/api/scope_config_api.go b/backend/plugins/asana/api/scope_config_api.go new file mode 100644 index 00000000000..d7e5b541b11 --- /dev/null +++ b/backend/plugins/asana/api/scope_config_api.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} + +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/asana/api/swagger.go b/backend/plugins/asana/api/swagger.go new file mode 100644 index 00000000000..43897adf9ba --- /dev/null +++ b/backend/plugins/asana/api/swagger.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +type AsanaTaskOptions tasks.AsanaOptions + +// @Summary asana task options for pipelines +// @Description Task options for asana pipelines +// @Tags plugins/asana +// @Accept application/json +// @Param pipeline body AsanaTaskOptions true "json" +// @Router /pipelines/asana/pipeline-task [post] +func _() {} diff --git a/backend/plugins/asana/asana.go b/backend/plugins/asana/asana.go new file mode 100644 index 00000000000..4bfbbf5b098 --- /dev/null +++ b/backend/plugins/asana/asana.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Asana //nolint + +func main() { + cmd := &cobra.Command{Use: "asana"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "asana connection id") + projectId := cmd.Flags().StringP("project", "p", "", "asana project gid") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("project") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "projectId": *projectId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/asana/e2e/e2e_path_test.go b/backend/plugins/asana/e2e/e2e_path_test.go new file mode 100644 index 00000000000..36858800890 --- /dev/null +++ b/backend/plugins/asana/e2e/e2e_path_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "path/filepath" + "runtime" +) + +// e2eDir returns the directory of the asana e2e package so CSV paths work when +// tests run from backend/ (CI) or from the e2e directory. +func e2eDir() string { + _, file, _, _ := runtime.Caller(0) + return filepath.Dir(file) +} + +func rawTablePath(name string) string { + return filepath.Join(e2eDir(), "raw_tables", name) +} + +func snapshotPath(name string) string { + return filepath.Join(e2eDir(), "snapshot_tables", name) +} diff --git a/backend/plugins/asana/e2e/project_test.go b/backend/plugins/asana/e2e/project_test.go new file mode 100644 index 00000000000..3fbc6308ea5 --- /dev/null +++ b/backend/plugins/asana/e2e/project_test.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaProjectDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + }, + } + + // Import raw data for projects + dataflowTester.ImportCsvIntoRawTable(rawTablePath("_raw_asana_projects.csv"), "_raw_asana_projects") + + // Verify project extraction + dataflowTester.FlushTabler(&models.AsanaProject{}) + dataflowTester.Subtask(tasks.ExtractProjectMeta, taskData) + + dataflowTester.VerifyTableWithOptions(models.AsanaProject{}, e2ehelper.TableOptions{ + CSVRelPath: snapshotPath("_tool_asana_projects.csv"), + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // Verify project conversion to domain layer board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertProjectMeta, taskData) + + dataflowTester.VerifyTable( + ticket.Board{}, + snapshotPath("boards.csv"), + []string{ + "id", + "name", + "description", + "url", + "created_date", + }, + ) +} + +func TestAsanaProjectWithScopeConfig(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "1234567890", + ScopeConfigId: 1, + }, + } + + // Import project with scope config association + dataflowTester.ImportCsvIntoRawTable(rawTablePath("_raw_asana_projects.csv"), "_raw_asana_projects") + dataflowTester.ImportCsvIntoTabler(snapshotPath("_tool_asana_scope_configs.csv"), &models.AsanaScopeConfig{}) + + // Extract project + dataflowTester.FlushTabler(&models.AsanaProject{}) + dataflowTester.Subtask(tasks.ExtractProjectMeta, taskData) + + // Verify project has scope_config_id + dataflowTester.VerifyTableWithOptions(models.AsanaProject{}, e2ehelper.TableOptions{ + CSVRelPath: snapshotPath("_tool_asana_projects_with_scope_config.csv"), + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/asana/e2e/raw_tables/_raw_asana_projects.csv b/backend/plugins/asana/e2e/raw_tables/_raw_asana_projects.csv new file mode 100644 index 00000000000..fcc941f85de --- /dev/null +++ b/backend/plugins/asana/e2e/raw_tables/_raw_asana_projects.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":""1234567890""}","{""gid"":""1234567890"",""name"":""Test Project"",""resource_type"":""project"",""archived"":false,""permalink_url"":""https://app.asana.com/0/1234567890"",""workspace"":{""gid"":""ws123""}}",https://app.asana.com/api/1.0/projects/1234567890,null,2025-02-03 12:00:00 diff --git a/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv b/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv new file mode 100644 index 00000000000..59815e7308c --- /dev/null +++ b/backend/plugins/asana/e2e/raw_tables/_raw_asana_tasks.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":""123456789""}","{""gid"":""987654321"",""name"":""Test Task"",""notes"":""Notes here"",""resource_type"":""task"",""resource_subtype"":""default_task"",""completed"":false,""due_on"":""2025-02-15"",""created_at"":""2025-02-01T10:00:00.000Z"",""permalink_url"":""https://app.asana.com/0/123/987654321"",""memberships"":[{""project"":{""gid"":""123456789""},""section"":{""gid"":""111""}}]}",https://app.asana.com/api/1.0/projects/123456789/tasks,null,2025-02-03 12:00:00 diff --git a/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects.csv b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects.csv new file mode 100644 index 00000000000..e05e3d2bad2 --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,gid,name,resource_type,archived,workspace_gid,permalink_url +1,0,1234567890,Test Project,project,0,ws123,https://app.asana.com/0/1234567890 diff --git a/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects_with_scope_config.csv b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects_with_scope_config.csv new file mode 100644 index 00000000000..12e7ee7aee3 --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_projects_with_scope_config.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,gid,name,resource_type,archived,workspace_gid,permalink_url +1,1,1234567890,Test Project,project,0,ws123,https://app.asana.com/0/1234567890 diff --git a/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_scope_configs.csv b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_scope_configs.csv new file mode 100644 index 00000000000..7fd96f9c494 --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_scope_configs.csv @@ -0,0 +1,2 @@ +id,connection_id,name,entities,issue_type_requirement,issue_type_bug,issue_type_incident +1,1,default,"[""TICKET""]",(feat|feature|story|requirement),(bug|defect|broken),(incident|outage|failure) diff --git a/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv new file mode 100644 index 00000000000..e4bde6d2b35 --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/_tool_asana_tasks.csv @@ -0,0 +1,2 @@ +connection_id,gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,project_gid,section_gid,section_name,assignee_gid,assignee_name,creator_gid,creator_name,parent_gid,num_subtasks,std_type,std_status,priority,story_point,severity,lead_time_minutes +1,987654321,Test Task,Notes here,task,default_task,0,,2025-02-15T00:00:00.000+00:00,2025-02-01T10:00:00.000+00:00,,https://app.asana.com/0/123/987654321,123456789,111,,,,,,,0,,,,,, diff --git a/backend/plugins/asana/e2e/snapshot_tables/boards.csv b/backend/plugins/asana/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..8dd0e8c6780 --- /dev/null +++ b/backend/plugins/asana/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date +asana:AsanaProject:1:1234567890,Test Project,,https://app.asana.com/0/1234567890, diff --git a/backend/plugins/asana/e2e/task_test.go b/backend/plugins/asana/e2e/task_test.go new file mode 100644 index 00000000000..5687ceb7c5b --- /dev/null +++ b/backend/plugins/asana/e2e/task_test.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/asana/impl" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +func TestAsanaTaskDataFlow(t *testing.T) { + var asana impl.Asana + dataflowTester := e2ehelper.NewDataFlowTester(t, "asana", asana) + + taskData := &tasks.AsanaTaskData{ + Options: &tasks.AsanaOptions{ + ConnectionId: 1, + ProjectId: "123456789", + }, + } + + dataflowTester.ImportCsvIntoRawTable(rawTablePath("_raw_asana_tasks.csv"), "_raw_asana_tasks") + + dataflowTester.FlushTabler(&models.AsanaTask{}) + dataflowTester.Subtask(tasks.ExtractTaskMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.AsanaTask{}, e2ehelper.TableOptions{ + CSVRelPath: snapshotPath("_tool_asana_tasks.csv"), + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/asana/impl/impl.go b/backend/plugins/asana/impl/impl.go new file mode 100644 index 00000000000..42ac786ae5f --- /dev/null +++ b/backend/plugins/asana/impl/impl.go @@ -0,0 +1,223 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/api" + "github.com/apache/incubator-devlake/plugins/asana/models" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/asana/tasks" +) + +var _ interface { + plugin.PluginTask + plugin.PluginMeta + plugin.PluginInit + plugin.PluginApi + plugin.PluginModel + plugin.PluginMigration + plugin.PluginSource + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Asana)(nil) + +type Asana struct{} + +func (p Asana) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Asana) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.AsanaConnection{}, + &models.AsanaProject{}, + &models.AsanaScopeConfig{}, + &models.AsanaTask{}, + &models.AsanaSection{}, + &models.AsanaUser{}, + &models.AsanaWorkspace{}, + &models.AsanaTeam{}, + &models.AsanaStory{}, + &models.AsanaTag{}, + &models.AsanaTaskTag{}, + &models.AsanaCustomField{}, + &models.AsanaTaskCustomFieldValue{}, + &models.AsanaProjectMembership{}, + &models.AsanaTeamMembership{}, + } +} + +func (p Asana) Description() string { + return "To collect and enrich data from Asana" +} + +func (p Asana) Name() string { + return "asana" +} + +func (p Asana) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + // Collect and extract in hierarchical order + // 1. Project (scope) + tasks.CollectProjectMeta, + tasks.ExtractProjectMeta, + // 2. Users (project members) + tasks.CollectUserMeta, + tasks.ExtractUserMeta, + // 3. Sections + tasks.CollectSectionMeta, + tasks.ExtractSectionMeta, + // 4. Tasks + tasks.CollectTaskMeta, + tasks.ExtractTaskMeta, + // 5. Subtasks (children of tasks) + tasks.CollectSubtaskMeta, + tasks.ExtractSubtaskMeta, + // 6. Stories (comments on tasks) + tasks.CollectStoryMeta, + tasks.ExtractStoryMeta, + // 7. Tags (on tasks) + tasks.CollectTagMeta, + tasks.ExtractTagMeta, + // Convert to domain layer + tasks.ConvertProjectMeta, + tasks.ConvertUserMeta, + tasks.ConvertTaskMeta, + tasks.ConvertStoryMeta, + } +} + +func (p Asana) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.AsanaOptions + err := helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "Asana plugin could not decode options") + } + if op.ProjectId == "" { + return nil, errors.BadInput.New("asana projectId is required") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("asana connectionId is invalid") + } + connection := &models.AsanaConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Asana plugin") + } + apiClient, err := tasks.CreateApiClient(taskCtx, connection) + if err != nil { + return nil, err + } + return &tasks.AsanaTaskData{ + Options: &op, + ApiClient: apiClient, + }, nil +} + +func (p Asana) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/asana" +} + +func (p Asana) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Asana) Connection() dal.Tabler { + return &models.AsanaConnection{} +} + +func (p Asana) Scope() plugin.ToolLayerScope { + return &models.AsanaProject{} +} + +func (p Asana) ScopeConfig() dal.Tabler { + return &models.AsanaScopeConfig{} +} + +func (p Asana) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Asana) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Asana) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.AsanaTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/asana/models/connection.go b/backend/plugins/asana/models/connection.go new file mode 100644 index 00000000000..349a69bcc2d --- /dev/null +++ b/backend/plugins/asana/models/connection.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// AsanaConn holds the essential information to connect to the Asana API +type AsanaConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (ac *AsanaConn) Sanitize() AsanaConn { + ac.Token = utils.SanitizeString(ac.Token) + return *ac +} + +// AsanaConnection holds AsanaConn plus ID/Name for database storage +type AsanaConnection struct { + helper.BaseConnection `mapstructure:",squash"` + AsanaConn `mapstructure:",squash"` +} + +func (connection *AsanaConnection) MergeFromRequest(target *AsanaConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (connection AsanaConnection) Sanitize() AsanaConnection { + connection.AsanaConn = connection.AsanaConn.Sanitize() + return connection +} + +// SetupAuthentication sets up the HTTP Request Authentication (Bearer token) +func (ac *AsanaConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", ac.Token)) + return nil +} + +func (AsanaConnection) TableName() string { + return "_tool_asana_connections" +} diff --git a/backend/plugins/asana/models/custom_field.go b/backend/plugins/asana/models/custom_field.go new file mode 100644 index 00000000000..aabade05f84 --- /dev/null +++ b/backend/plugins/asana/models/custom_field.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaCustomField represents a custom field definition +type AsanaCustomField struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Type string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + Precision int `json:"precision"` + IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` + HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` + common.NoPKModel +} + +func (AsanaCustomField) TableName() string { + return "_tool_asana_custom_fields" +} + +// AsanaTaskCustomFieldValue represents a custom field value on a task +type AsanaTaskCustomFieldValue struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldName string `gorm:"type:varchar(255)"` + DisplayValue string `gorm:"type:text"` + TextValue string `gorm:"type:text"` + NumberValue *float64 `json:"numberValue"` + EnumValueGid string `gorm:"type:varchar(255)"` + EnumValueName string `gorm:"type:varchar(255)"` + common.NoPKModel +} + +func (AsanaTaskCustomFieldValue) TableName() string { + return "_tool_asana_task_custom_field_values" +} diff --git a/backend/plugins/asana/models/membership.go b/backend/plugins/asana/models/membership.go new file mode 100644 index 00000000000..572e2464ecd --- /dev/null +++ b/backend/plugins/asana/models/membership.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaProjectMembership links users to projects with their role +type AsanaProjectMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + Role string `gorm:"type:varchar(32)"` + common.NoPKModel +} + +func (AsanaProjectMembership) TableName() string { + return "_tool_asana_project_memberships" +} + +// AsanaTeamMembership links users to teams +type AsanaTeamMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + IsGuest bool `json:"isGuest"` + common.NoPKModel +} + +func (AsanaTeamMembership) TableName() string { + return "_tool_asana_team_memberships" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go new file mode 100644 index 00000000000..a2dae698bd0 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250203_add_init_tables.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.AsanaConnection{}, + &archived.AsanaProject{}, + &archived.AsanaScopeConfig{}, + &archived.AsanaTask{}, + &archived.AsanaSection{}, + &archived.AsanaUser{}, + &archived.AsanaWorkspace{}, + &archived.AsanaTeam{}, + &archived.AsanaStory{}, + &archived.AsanaTag{}, + &archived.AsanaTaskTag{}, + &archived.AsanaCustomField{}, + &archived.AsanaTaskCustomFieldValue{}, + &archived.AsanaProjectMembership{}, + &archived.AsanaTeamMembership{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20250203000001 +} + +func (*addInitTables) Name() string { + return "asana init schemas" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go new file mode 100644 index 00000000000..35c482d271c --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_missing_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/asana/models/migrationscripts/archived" +) + +type addMissingTables struct{} + +func (*addMissingTables) Up(basicRes context.BasicRes) errors.Error { + // Add all the new tables that were added after the initial migration + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.AsanaWorkspace{}, + &archived.AsanaTeam{}, + &archived.AsanaStory{}, + &archived.AsanaTag{}, + &archived.AsanaTaskTag{}, + &archived.AsanaCustomField{}, + &archived.AsanaTaskCustomFieldValue{}, + &archived.AsanaProjectMembership{}, + &archived.AsanaTeamMembership{}, + ) +} + +func (*addMissingTables) Version() uint64 { + return 20250212000002 +} + +func (*addMissingTables) Name() string { + return "asana add missing tables for hierarchical data" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go new file mode 100644 index 00000000000..e092290c6ad --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_scope_config_issue_type_fields.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addScopeConfigIssueTypeFields)(nil) + +type addScopeConfigIssueTypeFields struct{} + +type asanaScopeConfig20250212v2 struct { + IssueTypeRequirement string `gorm:"type:varchar(255)"` + IssueTypeBug string `gorm:"type:varchar(255)"` + IssueTypeIncident string `gorm:"type:varchar(255)"` +} + +func (asanaScopeConfig20250212v2) TableName() string { + return "_tool_asana_scope_configs" +} + +func (*addScopeConfigIssueTypeFields) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + return db.AutoMigrate(&asanaScopeConfig20250212v2{}) +} + +func (*addScopeConfigIssueTypeFields) Version() uint64 { + return 20250212000004 +} + +func (*addScopeConfigIssueTypeFields) Name() string { + return "asana add issue type fields to scope config" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go new file mode 100644 index 00000000000..c8706d32c9f --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_task_transformation_fields.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addTaskTransformationFields)(nil) + +type addTaskTransformationFields struct{} + +type asanaTask20250212 struct { + SectionName string `gorm:"type:varchar(255)"` + StdType string `gorm:"type:varchar(255)"` + StdStatus string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + StoryPoint *float64 `gorm:"type:float8"` + Severity string `gorm:"type:varchar(255)"` + LeadTimeMinutes *uint +} + +func (asanaTask20250212) TableName() string { + return "_tool_asana_tasks" +} + +type asanaScopeConfig20250212 struct { + // Regex patterns for tag-based type classification (like GitHub) + IssueTypeRequirement string `gorm:"type:varchar(255)"` + IssueTypeBug string `gorm:"type:varchar(255)"` + IssueTypeIncident string `gorm:"type:varchar(255)"` +} + +func (asanaScopeConfig20250212) TableName() string { + return "_tool_asana_scope_configs" +} + +func (*addTaskTransformationFields) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + // Add transformation fields to tasks table + if err := db.AutoMigrate(&asanaTask20250212{}); err != nil { + return err + } + // Add transformation config fields to scope_configs table + return db.AutoMigrate(&asanaScopeConfig20250212{}) +} + +func (*addTaskTransformationFields) Version() uint64 { + return 20250212000003 +} + +func (*addTaskTransformationFields) Name() string { + return "asana add task transformation fields for issue tracking" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go new file mode 100644 index 00000000000..fc87c9fa1aa --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250212_add_user_photo_url.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addUserPhotoUrl)(nil) + +type addUserPhotoUrl struct{} + +type asanaUser20250212 struct { + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` +} + +func (asanaUser20250212) TableName() string { + return "_tool_asana_users" +} + +func (*addUserPhotoUrl) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + // Add photo_url and workspace_gids columns to _tool_asana_users table + return db.AutoMigrate(&asanaUser20250212{}) +} + +func (*addUserPhotoUrl) Version() uint64 { + return 20250212000001 +} + +func (*addUserPhotoUrl) Name() string { + return "asana add photo_url and workspace_gids to users table" +} diff --git a/backend/plugins/asana/models/migrationscripts/20250219_add_connection_id_to_scope_configs.go b/backend/plugins/asana/models/migrationscripts/20250219_add_connection_id_to_scope_configs.go new file mode 100644 index 00000000000..dff72ee5c15 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20250219_add_connection_id_to_scope_configs.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addConnectionIdToAsanaScopeConfigs)(nil) + +// asanaScopeConfig20250219 adds connection_id and name to match common.ScopeConfig. +// The init migration used archived.ScopeConfig which did not have these columns. +type asanaScopeConfig20250219 struct { + ConnectionId uint64 `json:"connectionId" gorm:"index" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex"` +} + +func (asanaScopeConfig20250219) TableName() string { + return "_tool_asana_scope_configs" +} + +type addConnectionIdToAsanaScopeConfigs struct{} + +func (*addConnectionIdToAsanaScopeConfigs) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().AutoMigrate(&asanaScopeConfig20250219{}) +} + +func (*addConnectionIdToAsanaScopeConfigs) Version() uint64 { + return 20250219000001 +} + +func (*addConnectionIdToAsanaScopeConfigs) Name() string { + return "add connection_id and name to _tool_asana_scope_configs" +} diff --git a/backend/plugins/asana/models/migrationscripts/20260509000001_encrypt_connection_token.go b/backend/plugins/asana/models/migrationscripts/20260509000001_encrypt_connection_token.go new file mode 100644 index 00000000000..6a67eaceead --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/20260509000001_encrypt_connection_token.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +type asanaConnectionTokenPlain struct { + ID uint64 `gorm:"primaryKey"` + Token string +} + +func (asanaConnectionTokenPlain) TableName() string { + return "_tool_asana_connections" +} + +type encryptConnectionToken struct{} + +func (*encryptConnectionToken) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + encKey := basicRes.GetConfig(plugin.EncodeKeyEnvStr) + if encKey == "" { + return errors.BadInput.New("asana invalid encKey") + } + + cursor, err := db.Cursor(dal.From(&asanaConnectionTokenPlain{})) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + row := &asanaConnectionTokenPlain{} + if err = db.Fetch(cursor, row); err != nil { + return err + } + if row.Token == "" { + continue + } + encryptedToken, err := plugin.Encrypt(encKey, row.Token) + if err != nil { + return err + } + err = db.UpdateColumns( + row.TableName(), + []dal.DalSet{{ColumnName: "token", Value: encryptedToken}}, + dal.Where("id = ?", row.ID), + ) + if err != nil { + return err + } + } + + return nil +} + +func (*encryptConnectionToken) Version() uint64 { + return 20260509000001 +} + +func (*encryptConnectionToken) Name() string { + return "encrypt asana connection token" +} diff --git a/backend/plugins/asana/models/migrationscripts/archived/models.go b/backend/plugins/asana/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..1551cad6115 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/archived/models.go @@ -0,0 +1,264 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type AsanaConnection struct { + archived.Model + Name string `gorm:"type:varchar(150);uniqueIndex" json:"name" validate:"required"` + Endpoint string `mapstructure:"endpoint" json:"endpoint" validate:"required"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec" encrypt:"yes"` +} + +func (AsanaConnection) TableName() string { + return "_tool_asana_connections" +} + +type AsanaProject struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `json:"gid" gorm:"type:varchar(255);primaryKey"` + Name string `json:"name" gorm:"type:varchar(255)"` + ResourceType string `json:"resourceType" gorm:"type:varchar(32)"` + Archived bool `json:"archived"` + WorkspaceGid string `json:"workspaceGid" gorm:"type:varchar(255)"` + PermalinkUrl string `json:"permalinkUrl" gorm:"type:varchar(512)"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId"` + archived.NoPKModel +} + +func (AsanaProject) TableName() string { + return "_tool_asana_projects" +} + +type AsanaScopeConfig struct { + archived.ScopeConfig + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (AsanaScopeConfig) TableName() string { + return "_tool_asana_scope_configs" +} + +type AsanaTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(512)"` + Notes string `gorm:"type:text"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completedAt"` + DueOn *time.Time `gorm:"type:date" json:"dueOn"` + CreatedAt time.Time `json:"createdAt"` + ModifiedAt *time.Time `json:"modifiedAt"` + PermalinkUrl string `gorm:"type:varchar(512)"` + ProjectGid string `gorm:"type:varchar(255);index"` + SectionGid string `gorm:"type:varchar(255);index"` + SectionName string `gorm:"type:varchar(255)"` + AssigneeGid string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatorGid string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + ParentGid string `gorm:"type:varchar(255);index"` + NumSubtasks int `json:"numSubtasks"` + StdType string `gorm:"type:varchar(255)"` + StdStatus string `gorm:"type:varchar(255)"` + Priority string `gorm:"type:varchar(255)"` + StoryPoint *float64 `json:"storyPoint"` + Severity string `gorm:"type:varchar(255)"` + LeadTimeMinutes *uint `json:"leadTimeMinutes"` + archived.NoPKModel +} + +func (AsanaTask) TableName() string { + return "_tool_asana_tasks" +} + +type AsanaSection struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ProjectGid string `gorm:"type:varchar(255);index"` + archived.NoPKModel +} + +func (AsanaSection) TableName() string { + return "_tool_asana_sections" +} + +type AsanaUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` + archived.NoPKModel +} + +func (AsanaUser) TableName() string { + return "_tool_asana_users" +} + +type AsanaWorkspace struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + IsOrganization bool `json:"isOrganization"` + archived.NoPKModel +} + +func (AsanaWorkspace) TableName() string { + return "_tool_asana_workspaces" +} + +type AsanaTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + HtmlDescription string `gorm:"type:text"` + OrganizationGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + archived.NoPKModel +} + +func (AsanaTeam) TableName() string { + return "_tool_asana_teams" +} + +type AsanaStory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(64)"` + Text string `gorm:"type:text"` + HtmlText string `gorm:"type:text"` + IsPinned bool `json:"isPinned"` + IsEdited bool `json:"isEdited"` + StickerName string `gorm:"type:varchar(64)"` + CreatedAt time.Time `json:"createdAt"` + CreatedByGid string `gorm:"type:varchar(255)"` + CreatedByName string `gorm:"type:varchar(255)"` + TaskGid string `gorm:"type:varchar(255);index"` + TargetGid string `gorm:"type:varchar(255);index"` + archived.NoPKModel +} + +func (AsanaStory) TableName() string { + return "_tool_asana_stories" +} + +type AsanaTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Color string `gorm:"type:varchar(32)"` + Notes string `gorm:"type:text"` + WorkspaceGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + archived.NoPKModel +} + +func (AsanaTag) TableName() string { + return "_tool_asana_tags" +} + +type AsanaTaskTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + TagGid string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (AsanaTaskTag) TableName() string { + return "_tool_asana_task_tags" +} + +type AsanaCustomField struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` + Type string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + Precision int `json:"precision"` + IsGlobalToWorkspace bool `json:"isGlobalToWorkspace"` + HasNotificationsEnabled bool `json:"hasNotificationsEnabled"` + archived.NoPKModel +} + +func (AsanaCustomField) TableName() string { + return "_tool_asana_custom_fields" +} + +type AsanaTaskCustomFieldValue struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldGid string `gorm:"primaryKey;type:varchar(255)"` + CustomFieldName string `gorm:"type:varchar(255)"` + DisplayValue string `gorm:"type:text"` + TextValue string `gorm:"type:text"` + NumberValue *float64 `json:"numberValue"` + EnumValueGid string `gorm:"type:varchar(255)"` + EnumValueName string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (AsanaTaskCustomFieldValue) TableName() string { + return "_tool_asana_task_custom_field_values" +} + +type AsanaProjectMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + Role string `gorm:"type:varchar(32)"` + archived.NoPKModel +} + +func (AsanaProjectMembership) TableName() string { + return "_tool_asana_project_memberships" +} + +type AsanaTeamMembership struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamGid string `gorm:"primaryKey;type:varchar(255)"` + UserGid string `gorm:"primaryKey;type:varchar(255)"` + IsGuest bool `json:"isGuest"` + archived.NoPKModel +} + +func (AsanaTeamMembership) TableName() string { + return "_tool_asana_team_memberships" +} diff --git a/backend/plugins/asana/models/migrationscripts/register.go b/backend/plugins/asana/models/migrationscripts/register.go new file mode 100644 index 00000000000..753f1926251 --- /dev/null +++ b/backend/plugins/asana/models/migrationscripts/register.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + new(addUserPhotoUrl), + new(addMissingTables), + new(addTaskTransformationFields), + new(addScopeConfigIssueTypeFields), + new(addConnectionIdToAsanaScopeConfigs), + new(encryptConnectionToken), + } +} diff --git a/backend/plugins/asana/models/project.go b/backend/plugins/asana/models/project.go new file mode 100644 index 00000000000..498bcd97ef1 --- /dev/null +++ b/backend/plugins/asana/models/project.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*AsanaProject)(nil) + +type AsanaProject struct { + common.Scope `mapstructure:",squash"` + Gid string `json:"gid" mapstructure:"gid" gorm:"type:varchar(255);primaryKey"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + ResourceType string `json:"resourceType" mapstructure:"resourceType" gorm:"type:varchar(32)"` + Archived bool `json:"archived" mapstructure:"archived"` + WorkspaceGid string `json:"workspaceGid" mapstructure:"workspaceGid" gorm:"type:varchar(255)"` + PermalinkUrl string `json:"permalinkUrl" mapstructure:"permalinkUrl" gorm:"type:varchar(512)"` +} + +func (p AsanaProject) ScopeId() string { + return p.Gid +} + +func (p AsanaProject) ScopeName() string { + return p.Name +} + +func (p AsanaProject) ScopeFullName() string { + return p.Name +} + +func (p AsanaProject) ScopeParams() interface{} { + return &AsanaApiParams{ + ConnectionId: p.ConnectionId, + ProjectId: p.Gid, + } +} + +func (AsanaProject) TableName() string { + return "_tool_asana_projects" +} + +type AsanaApiParams struct { + ConnectionId uint64 + ProjectId string +} diff --git a/backend/plugins/asana/models/scope_config.go b/backend/plugins/asana/models/scope_config.go new file mode 100644 index 00000000000..560e645a5e5 --- /dev/null +++ b/backend/plugins/asana/models/scope_config.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + + // Issue type mapping using regex patterns (like GitHub) + // Tags matching these patterns will classify the task type + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (AsanaScopeConfig) TableName() string { + return "_tool_asana_scope_configs" +} + +func (a *AsanaScopeConfig) SetConnectionId(c *AsanaScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/asana/models/section.go b/backend/plugins/asana/models/section.go new file mode 100644 index 00000000000..3ebc17b8a1b --- /dev/null +++ b/backend/plugins/asana/models/section.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaSection struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ProjectGid string `gorm:"type:varchar(255);index"` + common.NoPKModel +} + +func (AsanaSection) TableName() string { + return "_tool_asana_sections" +} diff --git a/backend/plugins/asana/models/story.go b/backend/plugins/asana/models/story.go new file mode 100644 index 00000000000..41386a41e24 --- /dev/null +++ b/backend/plugins/asana/models/story.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// AsanaStory represents comments and system-generated stories on tasks +type AsanaStory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(64)"` + Text string `gorm:"type:text"` + HtmlText string `gorm:"type:text"` + IsPinned bool `json:"isPinned"` + IsEdited bool `json:"isEdited"` + StickerName string `gorm:"type:varchar(64)"` + CreatedAt time.Time `json:"createdAt"` + CreatedByGid string `gorm:"type:varchar(255)"` + CreatedByName string `gorm:"type:varchar(255)"` + TaskGid string `gorm:"type:varchar(255);index"` + TargetGid string `gorm:"type:varchar(255);index"` + common.NoPKModel +} + +func (AsanaStory) TableName() string { + return "_tool_asana_stories" +} diff --git a/backend/plugins/asana/models/tag.go b/backend/plugins/asana/models/tag.go new file mode 100644 index 00000000000..2ab29555ad2 --- /dev/null +++ b/backend/plugins/asana/models/tag.go @@ -0,0 +1,50 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Color string `gorm:"type:varchar(32)"` + Notes string `gorm:"type:text"` + WorkspaceGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + common.NoPKModel +} + +func (AsanaTag) TableName() string { + return "_tool_asana_tags" +} + +// AsanaTaskTag is a many-to-many relationship between tasks and tags +type AsanaTaskTag struct { + ConnectionId uint64 `gorm:"primaryKey"` + TaskGid string `gorm:"primaryKey;type:varchar(255)"` + TagGid string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (AsanaTaskTag) TableName() string { + return "_tool_asana_task_tags" +} diff --git a/backend/plugins/asana/models/task.go b/backend/plugins/asana/models/task.go new file mode 100644 index 00000000000..fce6bd9388b --- /dev/null +++ b/backend/plugins/asana/models/task.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTask struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(512)"` + Notes string `gorm:"type:text"` + ResourceType string `gorm:"type:varchar(32)"` + ResourceSubtype string `gorm:"type:varchar(32)"` // default_task, milestone, section, approval + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completedAt"` + DueOn *time.Time `gorm:"type:date" json:"dueOn"` + CreatedAt time.Time `json:"createdAt"` + ModifiedAt *time.Time `json:"modifiedAt"` + PermalinkUrl string `gorm:"type:varchar(512)"` + ProjectGid string `gorm:"type:varchar(255);index"` + SectionGid string `gorm:"type:varchar(255);index"` + SectionName string `gorm:"type:varchar(255)"` // For status mapping + AssigneeGid string `gorm:"type:varchar(255)"` + AssigneeName string `gorm:"type:varchar(255)"` + CreatorGid string `gorm:"type:varchar(255)"` + CreatorName string `gorm:"type:varchar(255)"` + ParentGid string `gorm:"type:varchar(255);index"` + NumSubtasks int `json:"numSubtasks"` + + // Transformed fields for domain layer + StdType string `gorm:"type:varchar(255)"` // Standard type: REQUIREMENT, BUG, INCIDENT, EPIC, TASK, SUBTASK + StdStatus string `gorm:"type:varchar(255)"` // Standard status: TODO, IN_PROGRESS, DONE + + // Custom field values (extracted during transformation) + Priority string `gorm:"type:varchar(255)"` // Priority from custom field + StoryPoint *float64 `json:"storyPoint"` // Story points from custom field + Severity string `gorm:"type:varchar(255)"` // Severity from custom field + + // Lead time tracking + LeadTimeMinutes *uint `json:"leadTimeMinutes"` + + common.NoPKModel +} + +func (AsanaTask) TableName() string { + return "_tool_asana_tasks" +} diff --git a/backend/plugins/asana/models/team.go b/backend/plugins/asana/models/team.go new file mode 100644 index 00000000000..1488a0fd00c --- /dev/null +++ b/backend/plugins/asana/models/team.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + Description string `gorm:"type:text"` + HtmlDescription string `gorm:"type:text"` + OrganizationGid string `gorm:"type:varchar(255);index"` + PermalinkUrl string `gorm:"type:varchar(512)"` + common.NoPKModel +} + +func (AsanaTeam) TableName() string { + return "_tool_asana_teams" +} diff --git a/backend/plugins/asana/models/user.go b/backend/plugins/asana/models/user.go new file mode 100644 index 00000000000..92681195945 --- /dev/null +++ b/backend/plugins/asana/models/user.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaUser struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + PhotoUrl string `gorm:"type:varchar(512)"` + WorkspaceGids string `gorm:"type:text"` // JSON array of workspace GIDs + common.NoPKModel +} + +func (AsanaUser) TableName() string { + return "_tool_asana_users" +} diff --git a/backend/plugins/asana/models/workspace.go b/backend/plugins/asana/models/workspace.go new file mode 100644 index 00000000000..101c249b358 --- /dev/null +++ b/backend/plugins/asana/models/workspace.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type AsanaWorkspace struct { + ConnectionId uint64 `gorm:"primaryKey"` + Gid string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + ResourceType string `gorm:"type:varchar(32)"` + IsOrganization bool `json:"isOrganization"` + common.NoPKModel +} + +func (AsanaWorkspace) TableName() string { + return "_tool_asana_workspaces" +} diff --git a/backend/plugins/asana/tasks/api_client.go b/backend/plugins/asana/tasks/api_client.go new file mode 100644 index 00000000000..25abd421bdc --- /dev/null +++ b/backend/plugins/asana/tasks/api_client.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.AsanaConnection) (*api.ApiAsyncClient, errors.Error) { + if connection.GetEndpoint() == "" { + connection.Endpoint = "https://app.asana.com/api/1.0/" + } + apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + asyncApiClient, err := api.CreateAsyncApiClient(taskCtx, apiClient, nil) + if err != nil { + return nil, err + } + return asyncApiClient, nil +} diff --git a/backend/plugins/asana/tasks/project_collector.go b/backend/plugins/asana/tasks/project_collector.go new file mode 100644 index 00000000000..5ae69a68d63 --- /dev/null +++ b/backend/plugins/asana/tasks/project_collector.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawProjectTable = "asana_projects" + +var _ plugin.SubTaskEntryPoint = CollectProject + +var CollectProjectMeta = plugin.SubTaskMeta{ + Name: "CollectProject", + EntryPoint: CollectProject, + EnabledByDefault: true, + Description: "Collect project data from Asana API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaDataWrapper struct { + Data json.RawMessage `json:"data"` +} + +func CollectProject(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawProjectTable, + }, + ApiClient: data.ApiClient, + UrlTemplate: "projects/{{ .Params.ProjectId }}", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaDataWrapper + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + if len(w.Data) == 0 { + return nil, nil + } + return []json.RawMessage{w.Data}, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/project_convertor.go b/backend/plugins/asana/tasks/project_convertor.go new file mode 100644 index 00000000000..cd6f89576ec --- /dev/null +++ b/backend/plugins/asana/tasks/project_convertor.go @@ -0,0 +1,80 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertProject + +var ConvertProjectMeta = plugin.SubTaskMeta{ + Name: "ConvertProject", + EntryPoint: ConvertProject, + EnabledByDefault: true, + Description: "Convert tool layer Asana projects into domain layer boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertProject(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawProjectTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + clauses := []dal.Clause{ + dal.From(&models.AsanaProject{}), + dal.Where("connection_id = ? AND gid = ?", connectionId, projectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + boardIdGen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaProject{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolProject := inputRow.(*models.AsanaProject) + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(toolProject.ConnectionId, toolProject.Gid)}, + Name: toolProject.Name, + Url: toolProject.PermalinkUrl, + Type: "asana", + } + return []interface{}{domainBoard}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/asana/tasks/project_extractor.go b/backend/plugins/asana/tasks/project_extractor.go new file mode 100644 index 00000000000..dba73b8a654 --- /dev/null +++ b/backend/plugins/asana/tasks/project_extractor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProject + +var ExtractProjectMeta = plugin.SubTaskMeta{ + Name: "ExtractProject", + EntryPoint: ExtractProject, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_projects", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiProject struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Archived bool `json:"archived"` + PermalinkUrl string `json:"permalink_url"` + Workspace *struct { + Gid string `json:"gid"` + } `json:"workspace"` +} + +func ExtractProject(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawProjectTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiProject := &asanaApiProject{} + err := errors.Convert(json.Unmarshal(resData.Data, apiProject)) + if err != nil { + return nil, err + } + workspaceGid := "" + if apiProject.Workspace != nil { + workspaceGid = apiProject.Workspace.Gid + } + toolProject := &models.AsanaProject{ + Gid: apiProject.Gid, + Name: apiProject.Name, + ResourceType: apiProject.ResourceType, + Archived: apiProject.Archived, + PermalinkUrl: apiProject.PermalinkUrl, + WorkspaceGid: workspaceGid, + } + toolProject.ConnectionId = taskData.Options.ConnectionId + toolProject.ScopeConfigId = taskData.Options.ScopeConfigId + return []interface{}{toolProject}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/section_collector.go b/backend/plugins/asana/tasks/section_collector.go new file mode 100644 index 00000000000..782f47a90b7 --- /dev/null +++ b/backend/plugins/asana/tasks/section_collector.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawSectionTable = "asana_sections" + +var _ plugin.SubTaskEntryPoint = CollectSection + +var CollectSectionMeta = plugin.SubTaskMeta{ + Name: "CollectSection", + EntryPoint: CollectSection, + EnabledByDefault: true, + Description: "Collect section data from Asana API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaListWrapper struct { + Data []json.RawMessage `json:"data"` +} + +func CollectSection(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawSectionTable, + }, + ApiClient: data.ApiClient, + UrlTemplate: "projects/{{ .Params.ProjectId }}/sections", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaListWrapper + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + return w.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/section_extractor.go b/backend/plugins/asana/tasks/section_extractor.go new file mode 100644 index 00000000000..3c6e90a9b55 --- /dev/null +++ b/backend/plugins/asana/tasks/section_extractor.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractSection + +var ExtractSectionMeta = plugin.SubTaskMeta{ + Name: "ExtractSection", + EntryPoint: ExtractSection, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_sections", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiSection struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Project *struct { + Gid string `json:"gid"` + } `json:"project"` +} + +func ExtractSection(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawSectionTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiSection := &asanaApiSection{} + err := errors.Convert(json.Unmarshal(resData.Data, apiSection)) + if err != nil { + return nil, err + } + projectGid := taskData.Options.ProjectId + if apiSection.Project != nil { + projectGid = apiSection.Project.Gid + } + toolSection := &models.AsanaSection{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiSection.Gid, + Name: apiSection.Name, + ResourceType: apiSection.ResourceType, + ProjectGid: projectGid, + } + return []interface{}{toolSection}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/story_collector.go b/backend/plugins/asana/tasks/story_collector.go new file mode 100644 index 00000000000..7fa8896900e --- /dev/null +++ b/backend/plugins/asana/tasks/story_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawStoryTable = "asana_stories" + +var _ plugin.SubTaskEntryPoint = CollectStory + +var CollectStoryMeta = plugin.SubTaskMeta{ + Name: "CollectStory", + EntryPoint: CollectStory, + EnabledByDefault: true, + Description: "Collect story/comment data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectStory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks for this project + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawStoryTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/stories", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,resource_type,resource_subtype,text,html_text,is_pinned,is_edited,sticker_name,created_at,created_by,target") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +type simpleTask struct { + Gid string +} diff --git a/backend/plugins/asana/tasks/story_convertor.go b/backend/plugins/asana/tasks/story_convertor.go new file mode 100644 index 00000000000..f5df43b2ab7 --- /dev/null +++ b/backend/plugins/asana/tasks/story_convertor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertStory + +var ConvertStoryMeta = plugin.SubTaskMeta{ + Name: "ConvertStory", + EntryPoint: ConvertStory, + EnabledByDefault: true, + Description: "Convert tool layer Asana stories into domain layer issue comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertStory(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawStoryTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + // Only convert comment-type stories (not system-generated ones) + clauses := []dal.Clause{ + dal.From(&models.AsanaStory{}), + dal.Join("LEFT JOIN _tool_asana_tasks ON _tool_asana_stories.task_gid = _tool_asana_tasks.gid AND _tool_asana_stories.connection_id = _tool_asana_tasks.connection_id"), + dal.Where("_tool_asana_stories.connection_id = ? AND _tool_asana_tasks.project_gid = ? AND _tool_asana_stories.resource_subtype = ?", + connectionId, projectId, "comment_added"), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + commentIdGen := didgen.NewDomainIdGenerator(&models.AsanaStory{}) + taskIdGen := didgen.NewDomainIdGenerator(&models.AsanaTask{}) + userIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaStory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolStory := inputRow.(*models.AsanaStory) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(toolStory.ConnectionId, toolStory.Gid)}, + IssueId: taskIdGen.Generate(toolStory.ConnectionId, toolStory.TaskGid), + Body: toolStory.Text, + CreatedDate: toolStory.CreatedAt, + } + if toolStory.CreatedByGid != "" { + domainComment.AccountId = userIdGen.Generate(toolStory.ConnectionId, toolStory.CreatedByGid) + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/asana/tasks/story_extractor.go b/backend/plugins/asana/tasks/story_extractor.go new file mode 100644 index 00000000000..e0240f2da88 --- /dev/null +++ b/backend/plugins/asana/tasks/story_extractor.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractStory + +var ExtractStoryMeta = plugin.SubTaskMeta{ + Name: "ExtractStory", + EntryPoint: ExtractStory, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_stories", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiStory struct { + Gid string `json:"gid"` + ResourceType string `json:"resource_type"` + ResourceSubtype string `json:"resource_subtype"` + Text string `json:"text"` + HtmlText string `json:"html_text"` + IsPinned bool `json:"is_pinned"` + IsEdited bool `json:"is_edited"` + StickerName string `json:"sticker_name"` + CreatedAt time.Time `json:"created_at"` + CreatedBy *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"created_by"` + Target *struct { + Gid string `json:"gid"` + } `json:"target"` +} + +func ExtractStory(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawStoryTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiStory := &asanaApiStory{} + err := errors.Convert(json.Unmarshal(resData.Data, apiStory)) + if err != nil { + return nil, err + } + + // Extract task GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + createdByGid := "" + createdByName := "" + if apiStory.CreatedBy != nil { + createdByGid = apiStory.CreatedBy.Gid + createdByName = apiStory.CreatedBy.Name + } + targetGid := "" + if apiStory.Target != nil { + targetGid = apiStory.Target.Gid + } + + toolStory := &models.AsanaStory{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiStory.Gid, + ResourceType: apiStory.ResourceType, + ResourceSubtype: apiStory.ResourceSubtype, + Text: apiStory.Text, + HtmlText: apiStory.HtmlText, + IsPinned: apiStory.IsPinned, + IsEdited: apiStory.IsEdited, + StickerName: apiStory.StickerName, + CreatedAt: apiStory.CreatedAt, + CreatedByGid: createdByGid, + CreatedByName: createdByName, + TaskGid: input.Gid, + TargetGid: targetGid, + } + return []interface{}{toolStory}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/subtask_collector.go b/backend/plugins/asana/tasks/subtask_collector.go new file mode 100644 index 00000000000..a2f22cd3a1a --- /dev/null +++ b/backend/plugins/asana/tasks/subtask_collector.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawSubtaskTable = "asana_subtasks" + +var _ plugin.SubTaskEntryPoint = CollectSubtask + +var CollectSubtaskMeta = plugin.SubTaskMeta{ + Name: "CollectSubtask", + EntryPoint: CollectSubtask, + EnabledByDefault: true, + Description: "Collect subtask data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectSubtask(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks that have subtasks + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ? AND num_subtasks > 0", + data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawSubtaskTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/subtasks", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,created_by,parent,num_subtasks,memberships.section,memberships.project") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/subtask_extractor.go b/backend/plugins/asana/tasks/subtask_extractor.go new file mode 100644 index 00000000000..0e0b310facb --- /dev/null +++ b/backend/plugins/asana/tasks/subtask_extractor.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractSubtask + +var ExtractSubtaskMeta = plugin.SubTaskMeta{ + Name: "ExtractSubtask", + EntryPoint: ExtractSubtask, + EnabledByDefault: true, + Description: "Extract raw subtask data into tool layer table _tool_asana_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractSubtask(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawSubtaskTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTask := &asanaApiTask{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTask)) + if err != nil { + return nil, err + } + + // Get parent GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + assigneeGid := "" + assigneeName := "" + if apiTask.Assignee != nil { + assigneeGid = apiTask.Assignee.Gid + assigneeName = apiTask.Assignee.Name + } + creatorGid := "" + creatorName := "" + if apiTask.CreatedBy != nil { + creatorGid = apiTask.CreatedBy.Gid + creatorName = apiTask.CreatedBy.Name + } + sectionGid := "" + projectGid := taskData.Options.ProjectId + for _, m := range apiTask.Memberships { + if m.Project != nil { + projectGid = m.Project.Gid + } + if m.Section != nil && m.Section.Gid != "" { + sectionGid = m.Section.Gid + break + } + } + + var dueOn *time.Time + if apiTask.DueOn != "" { + dueOn = parseAsanaDate(apiTask.DueOn) + } + + toolTask := &models.AsanaTask{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTask.Gid, + Name: apiTask.Name, + Notes: apiTask.Notes, + ResourceType: apiTask.ResourceType, + ResourceSubtype: apiTask.ResourceSubtype, + Completed: apiTask.Completed, + CompletedAt: apiTask.CompletedAt, + DueOn: dueOn, + CreatedAt: apiTask.CreatedAt, + ModifiedAt: apiTask.ModifiedAt, + PermalinkUrl: apiTask.PermalinkUrl, + ProjectGid: projectGid, + SectionGid: sectionGid, + AssigneeGid: assigneeGid, + AssigneeName: assigneeName, + CreatorGid: creatorGid, + CreatorName: creatorName, + ParentGid: input.Gid, // Parent is the task that has subtasks + NumSubtasks: apiTask.NumSubtasks, + } + return []interface{}{toolTask}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/tag_collector.go b/backend/plugins/asana/tasks/tag_collector.go new file mode 100644 index 00000000000..0f3d218664e --- /dev/null +++ b/backend/plugins/asana/tasks/tag_collector.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawTagTable = "asana_tags" + +var _ plugin.SubTaskEntryPoint = CollectTag + +var CollectTagMeta = plugin.SubTaskMeta{ + Name: "CollectTag", + EntryPoint: CollectTag, + EnabledByDefault: true, + Description: "Collect tag data from Asana API for each task", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTag(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // Get all tasks for this project + clauses := []dal.Clause{ + dal.Select("gid"), + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(simpleTask{})) + if err != nil { + return err + } + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawTagTable, + }, + ApiClient: data.ApiClient, + Input: iterator, + UrlTemplate: "tasks/{{ .Input.Gid }}/tags", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,resource_type,color,notes,permalink_url") + query.Set("limit", "100") + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/tag_extractor.go b/backend/plugins/asana/tasks/tag_extractor.go new file mode 100644 index 00000000000..3e7db18ac4c --- /dev/null +++ b/backend/plugins/asana/tasks/tag_extractor.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTag + +var ExtractTagMeta = plugin.SubTaskMeta{ + Name: "ExtractTag", + EntryPoint: ExtractTag, + EnabledByDefault: true, + Description: "Extract raw data into tool layer tables _tool_asana_tags and _tool_asana_task_tags", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiTag struct { + Gid string `json:"gid"` + Name string `json:"name"` + ResourceType string `json:"resource_type"` + Color string `json:"color"` + Notes string `json:"notes"` + PermalinkUrl string `json:"permalink_url"` +} + +func ExtractTag(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawTagTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTag := &asanaApiTag{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTag)) + if err != nil { + return nil, err + } + + // Get task GID from input + var input struct { + Gid string `json:"gid"` + } + if err := errors.Convert(json.Unmarshal(resData.Input, &input)); err != nil { + return nil, err + } + + toolTag := &models.AsanaTag{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTag.Gid, + Name: apiTag.Name, + ResourceType: apiTag.ResourceType, + Color: apiTag.Color, + Notes: apiTag.Notes, + PermalinkUrl: apiTag.PermalinkUrl, + } + + // Create the task-tag relationship + taskTag := &models.AsanaTaskTag{ + ConnectionId: taskData.Options.ConnectionId, + TaskGid: input.Gid, + TagGid: apiTag.Gid, + } + + return []interface{}{toolTag, taskTag}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/task_collector.go b/backend/plugins/asana/tasks/task_collector.go new file mode 100644 index 00000000000..8e1b72c138b --- /dev/null +++ b/backend/plugins/asana/tasks/task_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawTaskTable = "asana_tasks" + +var _ plugin.SubTaskEntryPoint = CollectTask + +var CollectTaskMeta = plugin.SubTaskMeta{ + Name: "CollectTask", + EntryPoint: CollectTask, + EnabledByDefault: true, + Description: "Collect task data from Asana API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaTaskListResponse struct { + Data []json.RawMessage `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} + +func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawTaskTable, + }, + ApiClient: data.ApiClient, + PageSize: 100, + UrlTemplate: "projects/{{ .Params.ProjectId }}/tasks", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("limit", "100") + // Request all fields needed for transformation including section name + query.Set("opt_fields", "gid,name,notes,resource_type,resource_subtype,completed,completed_at,due_on,created_at,modified_at,permalink_url,assignee,assignee.name,created_by,created_by.name,parent,num_subtasks,memberships.section,memberships.section.name,memberships.project") + if reqData.CustomData != nil { + if offset, ok := reqData.CustomData.(string); ok && offset != "" { + query.Set("offset", offset) + } + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp asanaTaskListResponse + err := api.UnmarshalResponse(prevPageResponse, &resp) + if err != nil { + return nil, err + } + if resp.NextPage != nil && resp.NextPage.Offset != "" { + return resp.NextPage.Offset, nil + } + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var w asanaTaskListResponse + err := api.UnmarshalResponse(res, &w) + if err != nil { + return nil, err + } + return w.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/asana/tasks/task_convertor.go b/backend/plugins/asana/tasks/task_convertor.go new file mode 100644 index 00000000000..7c981b50d20 --- /dev/null +++ b/backend/plugins/asana/tasks/task_convertor.go @@ -0,0 +1,298 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertTask + +var ConvertTaskMeta = plugin.SubTaskMeta{ + Name: "ConvertTask", + EntryPoint: ConvertTask, + EnabledByDefault: true, + Description: "Convert tool layer Asana tasks into domain layer issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawTaskTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + projectId := data.Options.ProjectId + + // Get scope config for transformation rules + scopeConfig := getScopeConfig(taskCtx) + + // Get tags for tasks + taskTags := getTaskTags(db, connectionId) + + clauses := []dal.Clause{ + dal.From(&models.AsanaTask{}), + dal.Where("connection_id = ? AND project_gid = ?", connectionId, projectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + taskIdGen := didgen.NewDomainIdGenerator(&models.AsanaTask{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.AsanaProject{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaTask{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolTask := inputRow.(*models.AsanaTask) + + // Get tags for this task + tags := taskTags[toolTask.Gid] + + // Map type and status using scope config and tags + stdType, stdStatus := getStdTypeAndStatus(toolTask, scopeConfig, tags) + + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: taskIdGen.Generate(toolTask.ConnectionId, toolTask.Gid)}, + IssueKey: toolTask.Gid, + Title: toolTask.Name, + Description: toolTask.Notes, + Url: toolTask.PermalinkUrl, + Type: stdType, + OriginalType: toolTask.ResourceSubtype, + Status: stdStatus, + OriginalStatus: getOriginalStatus(toolTask), + StoryPoint: toolTask.StoryPoint, + CreatedDate: &toolTask.CreatedAt, + UpdatedDate: toolTask.ModifiedAt, + ResolutionDate: toolTask.CompletedAt, + DueDate: toolTask.DueOn, + CreatorName: toolTask.CreatorName, + AssigneeName: toolTask.AssigneeName, + LeadTimeMinutes: toolTask.LeadTimeMinutes, + } + + // Set creator and assignee IDs + if toolTask.CreatorGid != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, toolTask.CreatorGid) + } + if toolTask.AssigneeGid != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, toolTask.AssigneeGid) + } + + // Set parent issue ID if this is a subtask + if toolTask.ParentGid != "" { + domainIssue.ParentIssueId = taskIdGen.Generate(connectionId, toolTask.ParentGid) + // If no type determined and has parent, it's a subtask + if stdType == "" || stdType == ticket.TASK { + domainIssue.Type = ticket.SUBTASK + } + } + + // Set subtask flag + domainIssue.IsSubtask = toolTask.ParentGid != "" + + var result []interface{} + result = append(result, domainIssue) + + // Create board issue relationship + boardId := boardIdGen.Generate(connectionId, toolTask.ProjectGid) + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + result = append(result, boardIssue) + + // Create issue assignee if assignee exists + if toolTask.AssigneeGid != "" { + issueAssignee := &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: toolTask.AssigneeName, + } + result = append(result, issueAssignee) + } + + return result, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// getScopeConfig retrieves the scope config for transformation rules +func getScopeConfig(taskCtx plugin.SubTaskContext) *models.AsanaScopeConfig { + logger := taskCtx.GetLogger() + if taskCtx.GetData() == nil { + logger.Info("getScopeConfig: taskCtx.GetData() is nil") + return nil + } + data := taskCtx.GetData().(*AsanaTaskData) + db := taskCtx.GetDal() + + // First try to get by ScopeConfigId from options + if data.Options.ScopeConfigId != 0 { + var scopeConfig models.AsanaScopeConfig + err := db.First(&scopeConfig, dal.Where("id = ?", data.Options.ScopeConfigId)) + if err == nil { + logger.Info("getScopeConfig: Found scope config by ID %d, IssueTypeRequirement=%s, IssueTypeBug=%s, IssueTypeIncident=%s", + data.Options.ScopeConfigId, scopeConfig.IssueTypeRequirement, scopeConfig.IssueTypeBug, scopeConfig.IssueTypeIncident) + return &scopeConfig + } + logger.Info("getScopeConfig: Failed to get scope config by ID %d: %v", data.Options.ScopeConfigId, err) + } else { + logger.Info("getScopeConfig: ScopeConfigId is 0, trying to get from project") + } + + // Try to get scope config from project's scope_config_id + var project models.AsanaProject + err := db.First(&project, dal.Where("connection_id = ? AND gid = ?", data.Options.ConnectionId, data.Options.ProjectId)) + if err != nil { + logger.Info("getScopeConfig: Failed to get project: %v", err) + return nil + } + + if project.ScopeConfigId != 0 { + var scopeConfig models.AsanaScopeConfig + err := db.First(&scopeConfig, dal.Where("id = ?", project.ScopeConfigId)) + if err == nil { + logger.Info("getScopeConfig: Found scope config from project, IssueTypeRequirement=%s, IssueTypeBug=%s, IssueTypeIncident=%s", + scopeConfig.IssueTypeRequirement, scopeConfig.IssueTypeBug, scopeConfig.IssueTypeIncident) + return &scopeConfig + } + logger.Info("getScopeConfig: Failed to get scope config from project: %v", err) + } else { + logger.Info("getScopeConfig: Project has no scope_config_id") + } + + return nil +} + +// getTaskTags retrieves all tags for tasks and returns a map of taskGid -> []tagName +func getTaskTags(db dal.Dal, connectionId uint64) map[string][]string { + result := make(map[string][]string) + + var taskTags []models.AsanaTaskTag + err := db.All(&taskTags, dal.Where("connection_id = ?", connectionId)) + if err != nil { + return result + } + + // Get all tag names + tagNames := make(map[string]string) + var tags []models.AsanaTag + err = db.All(&tags, dal.Where("connection_id = ?", connectionId)) + if err == nil { + for _, tag := range tags { + tagNames[tag.Gid] = tag.Name + } + } + + // Build taskGid -> []tagName map + for _, tt := range taskTags { + if tagName, ok := tagNames[tt.TagGid]; ok { + result[tt.TaskGid] = append(result[tt.TaskGid], tagName) + } + } + + return result +} + +// getStdTypeAndStatus maps Asana task to standard type and status using regex patterns (like GitHub) +func getStdTypeAndStatus(task *models.AsanaTask, scopeConfig *models.AsanaScopeConfig, tags []string) (string, string) { + stdType := ticket.TASK + stdStatus := ticket.TODO + + // Default status based on completion + if task.Completed { + stdStatus = ticket.DONE + } + + // If no scope config, return defaults + if scopeConfig == nil { + return getDefaultType(task), stdStatus + } + + // Combine all tags into a single string for matching + tagString := strings.ToLower(strings.Join(tags, " ")) + + // Match issue type using regex patterns (like GitHub) + if scopeConfig.IssueTypeRequirement != "" && matchPattern(tagString, scopeConfig.IssueTypeRequirement) { + stdType = ticket.REQUIREMENT + } + if scopeConfig.IssueTypeBug != "" && matchPattern(tagString, scopeConfig.IssueTypeBug) { + stdType = ticket.BUG + } + if scopeConfig.IssueTypeIncident != "" && matchPattern(tagString, scopeConfig.IssueTypeIncident) { + stdType = ticket.INCIDENT + } + + // If no type matched and task is a subtask, mark it as subtask + if stdType == ticket.TASK && task.ParentGid != "" { + stdType = ticket.SUBTASK + } + + return stdType, stdStatus +} + +// getDefaultType returns the default type based on task properties +func getDefaultType(task *models.AsanaTask) string { + if task.ParentGid != "" { + return ticket.SUBTASK + } + return ticket.TASK +} + +// matchPattern checks if the input string matches the regex pattern +func matchPattern(input, pattern string) bool { + if pattern == "" { + return false + } + re, err := regexp.Compile("(?i)" + pattern) + if err != nil { + return false + } + return re.MatchString(input) +} + +// getOriginalStatus returns the original status string +func getOriginalStatus(task *models.AsanaTask) string { + if task.Completed { + return "completed" + } + if task.SectionName != "" { + return task.SectionName + } + return "incomplete" +} diff --git a/backend/plugins/asana/tasks/task_data.go b/backend/plugins/asana/tasks/task_data.go new file mode 100644 index 00000000000..b6c82836b7b --- /dev/null +++ b/backend/plugins/asana/tasks/task_data.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +func CreateRawDataSubTaskArgs(taskCtx plugin.SubTaskContext, rawTable string) (*api.RawDataSubTaskArgs, *AsanaTaskData) { + data := taskCtx.GetData().(*AsanaTaskData) + params := models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + } + return &api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: params, + Table: rawTable, + }, data +} + +type AsanaOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` + ProjectId string `json:"projectId" mapstructure:"projectId"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` +} + +type AsanaTaskData struct { + Options *AsanaOptions + ApiClient *api.ApiAsyncClient + Project *models.AsanaProject +} diff --git a/backend/plugins/asana/tasks/task_extractor.go b/backend/plugins/asana/tasks/task_extractor.go new file mode 100644 index 00000000000..11ce45af1e0 --- /dev/null +++ b/backend/plugins/asana/tasks/task_extractor.go @@ -0,0 +1,162 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTask + +var ExtractTaskMeta = plugin.SubTaskMeta{ + Name: "ExtractTask", + EntryPoint: ExtractTask, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type asanaApiTask struct { + Gid string `json:"gid"` + Name string `json:"name"` + Notes string `json:"notes"` + ResourceType string `json:"resource_type"` + ResourceSubtype string `json:"resource_subtype"` + Completed bool `json:"completed"` + CompletedAt *time.Time `json:"completed_at"` + DueOn string `json:"due_on"` + CreatedAt time.Time `json:"created_at"` + ModifiedAt *time.Time `json:"modified_at"` + PermalinkUrl string `json:"permalink_url"` + Assignee *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"assignee"` + CreatedBy *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"created_by"` + Parent *struct { + Gid string `json:"gid"` + } `json:"parent"` + NumSubtasks int `json:"num_subtasks"` + Memberships []struct { + Section *struct { + Gid string `json:"gid"` + Name string `json:"name"` + } `json:"section"` + Project *struct { + Gid string `json:"gid"` + } `json:"project"` + } `json:"memberships"` +} + +func parseAsanaDate(s string) *time.Time { + if s == "" { + return nil + } + t, err := time.Parse("2006-01-02", s) + if err != nil { + return nil + } + return &t +} + +func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawTaskTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiTask := &asanaApiTask{} + err := errors.Convert(json.Unmarshal(resData.Data, apiTask)) + if err != nil { + return nil, err + } + assigneeGid := "" + assigneeName := "" + if apiTask.Assignee != nil { + assigneeGid = apiTask.Assignee.Gid + assigneeName = apiTask.Assignee.Name + } + creatorGid := "" + creatorName := "" + if apiTask.CreatedBy != nil { + creatorGid = apiTask.CreatedBy.Gid + creatorName = apiTask.CreatedBy.Name + } + parentGid := "" + if apiTask.Parent != nil { + parentGid = apiTask.Parent.Gid + } + sectionGid := "" + sectionName := "" + projectGid := taskData.Options.ProjectId + for _, m := range apiTask.Memberships { + if m.Project != nil { + projectGid = m.Project.Gid + } + if m.Section != nil && m.Section.Gid != "" { + sectionGid = m.Section.Gid + sectionName = m.Section.Name + break + } + } + toolTask := &models.AsanaTask{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiTask.Gid, + Name: apiTask.Name, + Notes: apiTask.Notes, + ResourceType: apiTask.ResourceType, + ResourceSubtype: apiTask.ResourceSubtype, + Completed: apiTask.Completed, + CompletedAt: apiTask.CompletedAt, + DueOn: parseAsanaDate(apiTask.DueOn), + CreatedAt: apiTask.CreatedAt, + ModifiedAt: apiTask.ModifiedAt, + PermalinkUrl: apiTask.PermalinkUrl, + ProjectGid: projectGid, + SectionGid: sectionGid, + SectionName: sectionName, + AssigneeGid: assigneeGid, + AssigneeName: assigneeName, + CreatorGid: creatorGid, + CreatorName: creatorName, + ParentGid: parentGid, + NumSubtasks: apiTask.NumSubtasks, + } + return []interface{}{toolTask}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/asana/tasks/user_collector.go b/backend/plugins/asana/tasks/user_collector.go new file mode 100644 index 00000000000..9b1649aa1fb --- /dev/null +++ b/backend/plugins/asana/tasks/user_collector.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +const rawUserTable = "asana_users" + +var _ plugin.SubTaskEntryPoint = CollectUser + +var CollectUserMeta = plugin.SubTaskMeta{ + Name: "CollectUser", + EntryPoint: CollectUser, + EnabledByDefault: true, + Description: "Collect user data from Asana API (project members)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func CollectUser(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*AsanaTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: rawUserTable, + }, + ApiClient: data.ApiClient, + PageSize: 100, + UrlTemplate: "projects/{{ .Params.ProjectId }}/members", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("opt_fields", "gid,name,email,resource_type,photo.image_128x128") + query.Set("limit", "100") + if reqData.CustomData != nil { + if offset, ok := reqData.CustomData.(string); ok && offset != "" { + query.Set("offset", offset) + } + } + return query, nil + }, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(prevPageResponse, &resp) + if err != nil { + return nil, err + } + if resp.NextPage != nil && resp.NextPage.Offset != "" { + return resp.NextPage.Offset, nil + } + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resp asanaListResponse + err := api.UnmarshalResponse(res, &resp) + if err != nil { + return nil, err + } + return resp.Data, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +type asanaListResponse struct { + Data []json.RawMessage `json:"data"` + NextPage *struct { + Offset string `json:"offset"` + Path string `json:"path"` + URI string `json:"uri"` + } `json:"next_page"` +} diff --git a/backend/plugins/asana/tasks/user_convertor.go b/backend/plugins/asana/tasks/user_convertor.go new file mode 100644 index 00000000000..4d9f3da0802 --- /dev/null +++ b/backend/plugins/asana/tasks/user_convertor.go @@ -0,0 +1,80 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertUser + +var ConvertUserMeta = plugin.SubTaskMeta{ + Name: "ConvertUser", + EntryPoint: ConvertUser, + EnabledByDefault: true, + Description: "Convert tool layer Asana users into domain layer accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func ConvertUser(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, rawUserTable) + db := taskCtx.GetDal() + connectionId := data.Options.ConnectionId + + clauses := []dal.Clause{ + dal.From(&models.AsanaUser{}), + dal.Where("connection_id = ?", connectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + accountIdGen := didgen.NewDomainIdGenerator(&models.AsanaUser{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AsanaUser{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolUser := inputRow.(*models.AsanaUser) + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{Id: accountIdGen.Generate(toolUser.ConnectionId, toolUser.Gid)}, + Email: toolUser.Email, + FullName: toolUser.Name, + UserName: toolUser.Name, + AvatarUrl: toolUser.PhotoUrl, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/asana/tasks/user_extractor.go b/backend/plugins/asana/tasks/user_extractor.go new file mode 100644 index 00000000000..763c5a2a766 --- /dev/null +++ b/backend/plugins/asana/tasks/user_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/asana/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractUser + +var ExtractUserMeta = plugin.SubTaskMeta{ + Name: "ExtractUser", + EntryPoint: ExtractUser, + EnabledByDefault: true, + Description: "Extract raw data into tool layer table _tool_asana_users", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +type asanaApiUser struct { + Gid string `json:"gid"` + Name string `json:"name"` + Email string `json:"email"` + ResourceType string `json:"resource_type"` + Photo *struct { + Image128x128 string `json:"image_128x128"` + } `json:"photo"` +} + +func ExtractUser(taskCtx plugin.SubTaskContext) errors.Error { + taskData := taskCtx.GetData().(*AsanaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.AsanaApiParams{ + ConnectionId: taskData.Options.ConnectionId, + ProjectId: taskData.Options.ProjectId, + }, + Table: rawUserTable, + }, + Extract: func(resData *api.RawData) ([]interface{}, errors.Error) { + apiUser := &asanaApiUser{} + err := errors.Convert(json.Unmarshal(resData.Data, apiUser)) + if err != nil { + return nil, err + } + photoUrl := "" + if apiUser.Photo != nil { + photoUrl = apiUser.Photo.Image128x128 + } + toolUser := &models.AsanaUser{ + ConnectionId: taskData.Options.ConnectionId, + Gid: apiUser.Gid, + Name: apiUser.Name, + Email: apiUser.Email, + ResourceType: apiUser.ResourceType, + PhotoUrl: photoUrl, + } + return []interface{}{toolUser}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go index 07d4b7627a3..9ac7669178e 100644 --- a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go +++ b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go @@ -49,7 +49,7 @@ func CollectRecords(taskCtx plugin.SubTaskContext) errors.Error { cursor, err := db.Cursor( dal.Select("azuredevops_id"), dal.From(models.AzuredevopsBuild{}.TableName()), - dal.Where("repository_id = ? and connection_id=? and result != ?", data.Options.RepositoryId, data.Options.ConnectionId, "failed"), + dal.Where("repository_id = ? and connection_id=? and result NOT IN ?", data.Options.RepositoryId, data.Options.ConnectionId, []string{"failed", "none"}), ) if err != nil { return err @@ -68,7 +68,7 @@ func CollectRecords(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/build/builds/{{ .Input.AzuredevopsId }}/Timeline?api-version=7.1", Query: BuildPaginator(true), ResponseParser: ParseRawMessageFromRecords, - AfterResponse: ignoreDeletedBuilds, // Ignore the 404 response if builds are deleted during the collection + AfterResponse: ignoreInvalidTimelineResponse, // Skip builds with missing/malformed timelines (e.g. YAML syntax errors) }) if err != nil { return err diff --git a/backend/plugins/azuredevops_go/tasks/shared.go b/backend/plugins/azuredevops_go/tasks/shared.go index 3ceae7c9853..318850077e8 100644 --- a/backend/plugins/azuredevops_go/tasks/shared.go +++ b/backend/plugins/azuredevops_go/tasks/shared.go @@ -18,14 +18,17 @@ limitations under the License. package tasks import ( + "bytes" "encoding/json" "fmt" + "io" + "net/http" + "net/url" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "net/http" - "net/url" ) // Build and TimeLine Record State and Result types can be found here: @@ -145,9 +148,32 @@ func change203To401(res *http.Response) errors.Error { return nil } -func ignoreDeletedBuilds(res *http.Response) errors.Error { - if res.StatusCode == http.StatusNotFound { +// ignoreInvalidTimelineResponse is an AfterResponse handler for the Timeline API. +// It skips builds whose timeline response is missing or unparseable (e.g. builds +// that failed due to a YAML syntax error never produce a usable timeline), instead +// of aborting the entire subtask. +func ignoreInvalidTimelineResponse(res *http.Response) errors.Error { + // Treat 404 (build deleted) and 204 (build has no timeline data) as + // graceful skips so the subtask continues instead of failing. + // The 204 guard must come before the body is read because a 204 response + // has an empty body by definition and would cause the JSON parser to fail. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusNoContent { return api.ErrIgnoreAndContinue } + + // Read the body so we can inspect it, then restore it for the ResponseParser. + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + if err != nil { + return api.ErrIgnoreAndContinue + } + res.Body = io.NopCloser(bytes.NewReader(body)) + + // An empty body or non-JSON body means the timeline is not available. + // Return ErrIgnoreAndContinue so the build is skipped without failing the subtask. + if len(body) == 0 || !json.Valid(body) { + return api.ErrIgnoreAndContinue + } + return nil } diff --git a/backend/plugins/azuredevops_go/tasks/shared_test.go b/backend/plugins/azuredevops_go/tasks/shared_test.go new file mode 100644 index 00000000000..d46e01d6c0d --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/shared_test.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func makeResponse(statusCode int, body string) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(bytes.NewBufferString(body)), + Request: &http.Request{}, + } +} + +func TestIgnoreInvalidTimelineResponse_404(t *testing.T) { + res := makeResponse(http.StatusNotFound, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "404 should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_204(t *testing.T) { + res := makeResponse(http.StatusNoContent, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "204 No Content should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_EmptyBody(t *testing.T) { + res := makeResponse(http.StatusOK, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "empty body should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_NonJSONBody(t *testing.T) { + res := makeResponse(http.StatusOK, "not json at all") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "non-JSON body should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_ValidJSON_ReturnsNil(t *testing.T) { + validJSON := `{"records":[{"id":"abc","type":"Job","name":"Build"}]}` + res := makeResponse(http.StatusOK, validJSON) + err := ignoreInvalidTimelineResponse(res) + assert.Nil(t, err, "valid JSON body should return nil") + + // Verify body is still readable after the handler restored it. + remaining, readErr := io.ReadAll(res.Body) + assert.NoError(t, readErr) + assert.Equal(t, validJSON, string(remaining), "body should be restored for downstream parser") +} + +func TestIgnoreInvalidTimelineResponse_ValidEmptyJSONObject(t *testing.T) { + res := makeResponse(http.StatusOK, "{}") + err := ignoreInvalidTimelineResponse(res) + assert.Nil(t, err, "valid empty JSON object should return nil") +} diff --git a/backend/plugins/bamboo/api/connection_api.go b/backend/plugins/bamboo/api/connection_api.go index 3fd72a22e48..4c142fc5d27 100644 --- a/backend/plugins/bamboo/api/connection_api.go +++ b/backend/plugins/bamboo/api/connection_api.go @@ -108,7 +108,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource // @Param body body models.BambooConnection true "json body" // @Success 200 {object} models.BambooConnection // @Failure 400 {string} errcode.Error "Bad Request" -// @Failure 500 {string} errcode.Error "Internel Error" +// @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/bamboo/connections [POST] func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.Post(input) @@ -121,7 +121,7 @@ func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Param connectionId path int true "connection ID" // @Success 200 {object} models.BambooConnection // @Failure 400 {string} errcode.Error "Bad Request" -// @Failure 500 {string} errcode.Error "Internel Error" +// @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/bamboo/connections/{connectionId} [PATCH] func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.Patch(input) @@ -134,7 +134,7 @@ func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Success 200 {object} models.BambooConnection // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" -// @Failure 500 {string} errcode.Error "Internel Error" +// @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/bamboo/connections/{connectionId} [DELETE] func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.Delete(input) @@ -145,7 +145,7 @@ func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput // @Tags plugins/bamboo // @Success 200 {object} []models.BambooConnection // @Failure 400 {string} errcode.Error "Bad Request" -// @Failure 500 {string} errcode.Error "Internel Error" +// @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/bamboo/connections [GET] func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.GetAll(input) @@ -157,7 +157,7 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Param connectionId path int true "connection ID" // @Success 200 {object} models.BambooConnection // @Failure 400 {string} errcode.Error "Bad Request" -// @Failure 500 {string} errcode.Error "Internel Error" +// @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/bamboo/connections/{connectionId} [GET] func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.GetDetail(input) diff --git a/backend/plugins/bamboo/models/plan.go b/backend/plugins/bamboo/models/plan.go index 3811596f119..386fbbbc433 100644 --- a/backend/plugins/bamboo/models/plan.go +++ b/backend/plugins/bamboo/models/plan.go @@ -110,7 +110,7 @@ type SearchEntity struct { Type string `json:"type"` } -// Name trys to keep plan's name field the same with name in /remote-scopes. +// Name tries to keep plan's name field the same with name in /remote-scopes. // In /remote-scopes, plan's name is "{projectName - planName}". func (entity SearchEntity) Name() string { return strings.Join([]string{entity.ProjectName, entity.PlanName}, " - ") diff --git a/backend/plugins/bamboo/tasks/shared.go b/backend/plugins/bamboo/tasks/shared.go index 182670b954d..fc715aa114f 100644 --- a/backend/plugins/bamboo/tasks/shared.go +++ b/backend/plugins/bamboo/tasks/shared.go @@ -139,7 +139,7 @@ func generateFakeRepoUrl(endpoint string, repoId int) (string, error) { return fmt.Sprintf("fake://%s/repos/%d", endpointURL.Host, repoId), nil } -// covertError will indentify some known errors and transform it to a simple form. +// covertError will identify some known errors and transform it to a simple form. func covertError(err errors.Error) errors.Error { if err == nil { return nil diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go b/backend/plugins/bitbucket/api/blueprint_v200.go index 6fedb0df298..2ee774457f2 100644 --- a/backend/plugins/bitbucket/api/blueprint_v200.go +++ b/backend/plugins/bitbucket/api/blueprint_v200.go @@ -75,8 +75,11 @@ func makeDataSourcePipelinePlanV200( scopeDetails []*srvhelper.ScopeDetail[models.BitbucketRepo, models.BitbucketScopeConfig], connection *models.BitbucketConnection, ) (coreModels.PipelinePlan, errors.Error) { - plan := make(coreModels.PipelinePlan, len(scopeDetails)) + plan := make(coreModels.PipelinePlan, 0, len(scopeDetails)+1) for i, scopeDetail := range scopeDetails { + if i == len(plan) { + plan = append(plan, nil) + } bitbucketRepo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig stage := plan[i] if stage == nil { diff --git a/backend/plugins/bitbucket/api/remote_api.go b/backend/plugins/bitbucket/api/remote_api.go index 2da3f7ab626..cf60b1bd51d 100644 --- a/backend/plugins/bitbucket/api/remote_api.go +++ b/backend/plugins/bitbucket/api/remote_api.go @@ -67,11 +67,15 @@ func listBitbucketWorkspaces( err errors.Error, ) { var res *http.Response + // /user/permissions/workspaces was removed and /workspaces deprecated by + // Bitbucket CHANGE-2770; /user/workspaces lists the current user's workspaces + // and is the supported replacement. res, err = apiClient.Get( - "/user/permissions/workspaces", + "/user/workspaces", url.Values{ - "sort": {"workspace.slug"}, - "fields": {"values.workspace.slug,values.workspace.name,pagelen,page,size"}, + // No sort/fields: /user/workspaces rejects sort=workspace.slug with + // HTTP 400 "Invalid field name". The bare call returns the nested + // workspace objects we need; WorkspaceResponse ignores extra fields. "page": {fmt.Sprintf("%v", page.Page)}, "pagelen": {fmt.Sprintf("%v", page.PageLen)}, }, @@ -80,6 +84,15 @@ func listBitbucketWorkspaces( if err != nil { return } + if res.StatusCode > 299 { + body, e := io.ReadAll(res.Body) + if e != nil { + err = errors.BadInput.Wrap(e, "failed to read response body") + return + } + err = errors.HttpStatus(res.StatusCode).New(string(body)) + return + } resBody := &models.WorkspaceResponse{} err = api.UnmarshalResponse(res, resBody) @@ -89,9 +102,9 @@ func listBitbucketWorkspaces( for _, r := range resBody.Values { children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ Type: api.RAS_ENTRY_TYPE_GROUP, - Id: r.Workspace.Slug, - Name: r.Workspace.Name, - FullName: r.Workspace.Name, + Id: r.GroupId(), + Name: r.GroupName(), + FullName: r.GroupName(), }) } return @@ -143,6 +156,10 @@ func listBitbucketRepos( return } +// searchBitbucketRepos searches repositories by name across the user's workspaces. +// The cross-workspace GET /repositories?role=member was removed by Bitbucket +// CHANGE-2770, so we enumerate workspaces and query the workspace-scoped +// GET /repositories/{workspace} endpoint for each, aggregating up to PageSize hits. func searchBitbucketRepos( apiClient plugin.ApiClient, params *dsmodels.DsRemoteApiScopeSearchParams, @@ -150,39 +167,84 @@ func searchBitbucketRepos( children []dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo], err errors.Error, ) { - var res *http.Response - res, err = apiClient.Get( - "/repositories", - url.Values{ - "sort": {"name"}, - "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, - "role": {"member"}, - "q": {fmt.Sprintf(`full_name~"%s"`, params.Search)}, - "page": {fmt.Sprintf("%v", params.Page)}, - "pagelen": {fmt.Sprintf("%v", params.PageSize)}, - }, - nil, - ) - if err != nil { - return nil, err + pageSize := params.PageSize + if pageSize == 0 { + pageSize = 100 } - var resBody models.ReposResponse - err = api.UnmarshalResponse(res, &resBody) + + workspaces, err := listAllBitbucketWorkspaces(apiClient) if err != nil { - return + return nil, err } - for _, r := range resBody.Values { - children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ - Type: api.RAS_ENTRY_TYPE_SCOPE, - Id: r.FullName, - Name: r.Name, - FullName: r.FullName, - Data: r.ConvertApiScope(), - }) + + for _, workspace := range workspaces { + if len(children) >= pageSize { + break + } + var res *http.Response + res, err = apiClient.Get( + fmt.Sprintf("/repositories/%s", workspace), + url.Values{ + "sort": {"name"}, + "fields": {"values.name,values.full_name,values.language,values.description,values.owner.display_name,values.created_on,values.updated_on,values.links.clone,values.links.html,pagelen,page,size"}, + "q": {fmt.Sprintf(`name~"%s"`, params.Search)}, + "pagelen": {fmt.Sprintf("%v", pageSize)}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.ReposResponse + err = api.UnmarshalResponse(res, &resBody) + if err != nil { + return + } + for _, r := range resBody.Values { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.BitbucketRepo]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: r.FullName, + Name: r.Name, + FullName: r.FullName, + Data: r.ConvertApiScope(), + }) + } } return } +// listAllBitbucketWorkspaces returns every workspace slug accessible to the +// authenticated user, following pagination of GET /2.0/user/workspaces. +func listAllBitbucketWorkspaces(apiClient plugin.ApiClient) ([]string, errors.Error) { + var slugs []string + for page := 1; ; page++ { + res, err := apiClient.Get( + "/user/workspaces", + url.Values{ + // No sort/fields (see listBitbucketWorkspaces): /user/workspaces + // returns 400 on sort=workspace.slug. + "page": {fmt.Sprintf("%v", page)}, + "pagelen": {"100"}, + }, + nil, + ) + if err != nil { + return nil, err + } + var resBody models.WorkspaceResponse + if err = api.UnmarshalResponse(res, &resBody); err != nil { + return nil, err + } + for _, w := range resBody.Values { + slugs = append(slugs, w.GroupId()) + } + if len(resBody.Values) == 0 || page*resBody.Pagelen >= resBody.Size { + break + } + } + return slugs, nil +} + // RemoteScopes list all available scopes on the remote server // @Summary list all available scopes on the remote server // @Description list all available scopes on the remote server diff --git a/backend/plugins/bitbucket/models/repo.go b/backend/plugins/bitbucket/models/repo.go index 799b549b923..b33df3615b6 100644 --- a/backend/plugins/bitbucket/models/repo.go +++ b/backend/plugins/bitbucket/models/repo.go @@ -117,17 +117,15 @@ type WorkspaceResponse struct { Values []GroupResponse `json:"values"` } +// GroupResponse maps an entry from GET /2.0/user/workspaces, the supported +// replacement after Bitbucket CHANGE-2770 removed the cross-workspace +// GET /2.0/user/permissions/workspaces and deprecated GET /2.0/workspaces. +// Each value nests the workspace slug/name under a "workspace" object. type GroupResponse struct { - //Type string `json:"type"` - //Permission string `json:"permission"` - //LastAccessed time.Time `json:"last_accessed"` - //AddedOn time.Time `json:"added_on"` Workspace WorkspaceItem `json:"workspace"` } type WorkspaceItem struct { - //Type string `json:"type"` - //Uuid string `json:"uuid"` Slug string `json:"slug" group:"id"` Name string `json:"name" group:"name"` } diff --git a/backend/plugins/bitbucket/tasks/api_common.go b/backend/plugins/bitbucket/tasks/api_common.go index 352f41cb952..ad21c9b1e65 100644 --- a/backend/plugins/bitbucket/tasks/api_common.go +++ b/backend/plugins/bitbucket/tasks/api_common.go @@ -238,7 +238,10 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error { if res.StatusCode == http.StatusUnauthorized { return errors.Unauthorized.New("authentication failed, please check your AccessToken") } - if res.StatusCode == http.StatusNotFound { + // 404: repo has no issue tracker. 410 Gone: Bitbucket has sunset the issue + // tracker/wiki API, so the endpoint is permanently removed for the repo. + // Both mean "nothing to collect" — skip gracefully instead of retrying. + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusGone { return api.ErrIgnoreAndContinue } return nil diff --git a/backend/plugins/bitbucket/tasks/api_common_test.go b/backend/plugins/bitbucket/tasks/api_common_test.go new file mode 100644 index 00000000000..9d15e52e9c4 --- /dev/null +++ b/backend/plugins/bitbucket/tasks/api_common_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestIgnoreHTTPStatus404(t *testing.T) { + cases := []struct { + name string + statusCode int + wantIgnore bool // expect ErrIgnoreAndContinue (graceful skip, no retry) + wantErr bool // expect a real error + }{ + {"404 no issue tracker -> ignore", http.StatusNotFound, true, false}, + {"410 issue tracker sunset -> ignore", http.StatusGone, true, false}, + {"401 unauthorized -> error", http.StatusUnauthorized, false, true}, + {"200 ok -> continue", http.StatusOK, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ignoreHTTPStatus404(&http.Response{StatusCode: tc.statusCode}) + switch { + case tc.wantIgnore: + assert.Equal(t, api.ErrIgnoreAndContinue, err) + case tc.wantErr: + assert.Error(t, err) + default: + assert.NoError(t, err) + } + }) + } +} diff --git a/backend/plugins/bitbucket_server/api/blueprint_v200.go b/backend/plugins/bitbucket_server/api/blueprint_v200.go index 04bd908143b..0b0ae74c641 100644 --- a/backend/plugins/bitbucket_server/api/blueprint_v200.go +++ b/backend/plugins/bitbucket_server/api/blueprint_v200.go @@ -75,8 +75,11 @@ func makeDataSourcePipelinePlanV200( scopeDetails []*srvhelper.ScopeDetail[models.BitbucketServerRepo, models.BitbucketServerScopeConfig], connection *models.BitbucketServerConnection, ) (coreModels.PipelinePlan, errors.Error) { - plan := make(coreModels.PipelinePlan, len(scopeDetails)) + plan := make(coreModels.PipelinePlan, 0, len(scopeDetails)+1) for i, scopeDetail := range scopeDetails { + if i == len(plan) { + plan = append(plan, nil) + } bitbucketRepo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig stage := plan[i] if stage == nil { diff --git a/backend/plugins/circleci/circleci b/backend/plugins/circleci/circleci new file mode 100755 index 00000000000..745e690b9f8 Binary files /dev/null and b/backend/plugins/circleci/circleci differ diff --git a/backend/plugins/circleci/e2e/incremental_test.go b/backend/plugins/circleci/e2e/incremental_test.go new file mode 100644 index 00000000000..85de8a7ba33 --- /dev/null +++ b/backend/plugins/circleci/e2e/incremental_test.go @@ -0,0 +1,154 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "sort" + "testing" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/circleci/impl" + "github.com/apache/incubator-devlake/plugins/circleci/models" + "github.com/apache/incubator-devlake/plugins/circleci/tasks" + "github.com/stretchr/testify/assert" +) + +// TestCircleciWorkflowIncremental verifies that the workflow collector's incremental +// logic correctly filters pipelines by created_date. This is a regression test for +// the created_at -> created_date column rename. +func TestCircleciWorkflowIncremental(t *testing.T) { + var circleci impl.Circleci + + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) + taskData := &tasks.CircleciTaskData{ + Options: &tasks.CircleciOptions{ + ConnectionId: 1, + ProjectSlug: "github/coldgust/coldgust.github.io", + }, + RegexEnricher: api.NewRegexEnricher(), + } + + // seed pipelines table via extraction + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_pipelines.csv", "_raw_circleci_api_pipelines") + dataflowTester.FlushTabler(&models.CircleciPipeline{}) + dataflowTester.Subtask(tasks.ExtractPipelinesMeta, taskData) + + // Part 1: verify the SQL query used by BuildInputIterator works with created_date. + // Pipelines #4-10 have created_date > 17:45 — assert the exact IDs returned. + createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC) + var pipelines []models.CircleciPipeline + assert.Nil(t, dataflowTester.Dal.All(&pipelines, + dal.Where("connection_id = ? AND project_slug = ? AND created_date > ?", + 1, "github/coldgust/coldgust.github.io", createdAfter), + )) + pipelineIds := make([]string, len(pipelines)) + for i, p := range pipelines { + pipelineIds[i] = p.Id + } + sort.Strings(pipelineIds) + assert.Equal(t, []string{ + "23622ee4-e150-4920-9d66-81533fa765a4", // pipeline #5 + "2c45280f-7fb3-4025-b703-a547c4a94916", // pipeline #4 + "70f3eb15-3b94-4f80-b65e-f23f4b74c33a", // pipeline #6 + "7fcc1623-edcc-4a76-ad20-cd81aa83519f", // pipeline #9 + "866e967d-f826-4470-aed6-fc0c92e98703", // pipeline #7 + "afe0cabe-e7ee-4eb7-bf13-bb6170d139f0", // pipeline #8 + "d323f088-02fa-4ed5-9696-fc2f89a27150", // pipeline #10 + }, pipelineIds) + + // Part 2: verify extraction with only the incrementally-collected workflow raw data + // (workflows for pipelines #4-9 only). + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows_incremental.csv", "_raw_circleci_api_workflows") + dataflowTester.FlushTabler(&models.CircleciWorkflow{}) + dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + models.CircleciWorkflow{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_circleci_workflows_incremental.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + IgnoreFields: []string{"started_at", "stopped_at"}, + }, + ) +} + +// TestCircleciJobIncremental verifies that the job collector's incremental logic +// correctly filters workflows by created_date. Regression test for the column rename. +func TestCircleciJobIncremental(t *testing.T) { + var circleci impl.Circleci + + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) + taskData := &tasks.CircleciTaskData{ + Options: &tasks.CircleciOptions{ + ConnectionId: 1, + ProjectSlug: "github/coldgust/coldgust.github.io", + }, + RegexEnricher: api.NewRegexEnricher(), + Project: &models.CircleciProject{ + Id: "abcd", + }, + } + + // seed workflows table via extraction (all 10 workflows including b3b77371 with null created_date) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows.csv", "_raw_circleci_api_workflows") + dataflowTester.FlushTabler(&models.CircleciWorkflow{}) + dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData) + + // Part 1: verify the SQL query used by BuildInputIterator works with created_date. + // Workflows for pipelines #4-9 have created_date > 17:45 — assert the exact IDs returned. + // Workflow b3b77371 (null created_date) is excluded by the > comparison. + createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC) + var workflows []models.CircleciWorkflow + assert.Nil(t, dataflowTester.Dal.All(&workflows, + dal.Where("connection_id = ? AND project_slug = ? AND created_date > ?", + 1, "github/coldgust/coldgust.github.io", createdAfter), + )) + workflowIds := make([]string, len(workflows)) + for i, w := range workflows { + workflowIds[i] = w.Id + } + sort.Strings(workflowIds) + assert.Equal(t, []string{ + "6731159f-5275-4bfa-ba70-39d343d63814", // pipeline #5 + "7370985a-9de3-4a47-acbc-e6a1fe8e5812", // pipeline #7 + "b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14", // pipeline #6 + "c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943", // pipeline #4 + "fc76deef-bcdd-4856-8e96-a8e2d1c5a85f", // pipeline #8 + "fd0bd4f5-264f-4e3c-a151-06153c018f78", // pipeline #9 + }, workflowIds) + + // Part 2: verify extraction with only the incrementally-collected job raw data + // (jobs for workflows from pipelines #4-9 only). + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_projects.csv", "_raw_circleci_api_projects") + dataflowTester.FlushTabler(&models.CircleciProject{}) + dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData) + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_jobs_incremental.csv", "_raw_circleci_api_jobs") + dataflowTester.FlushTabler(&models.CircleciJob{}) + dataflowTester.Subtask(tasks.ExtractJobsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + models.CircleciJob{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_circleci_jobs_incremental.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }, + ) +} diff --git a/backend/plugins/circleci/e2e/job_collector_test.go b/backend/plugins/circleci/e2e/job_collector_test.go new file mode 100644 index 00000000000..ed2f014217f --- /dev/null +++ b/backend/plugins/circleci/e2e/job_collector_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "reflect" + "sort" + "testing" + + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/circleci/impl" + "github.com/apache/incubator-devlake/plugins/circleci/models" + "github.com/apache/incubator-devlake/plugins/circleci/tasks" + "github.com/stretchr/testify/assert" +) + +// TestCircleciUnfinishedJobsInputIterator is a regression test for +// https://github.com/apache/devlake/issues/8907. The "collect unfinished job +// details" collector builds its URL from "/v2/workflow/{{ .Input.Id }}/job" while +// scanning rows into a models.CircleciJob. Its input query must therefore expose the +// workflow id in the row's Id field; a bare "DISTINCT workflow_id" left Id empty and +// produced "/v2/workflow//job" (HTTP 500). This test runs the production query +// (tasks.UnfinishedJobsInputClauses) through the real iterator and asserts each +// yielded row's Id is the workflow id, that results are DISTINCT, and that the +// status/connection filters hold. +func TestCircleciUnfinishedJobsInputIterator(t *testing.T) { + var circleci impl.Circleci + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) + + const projectSlug = "github/test/repo" + dataflowTester.FlushTabler(&models.CircleciJob{}) + + seed := []models.CircleciJob{ + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-1", ProjectSlug: projectSlug, Status: "on_hold"}, + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-2", ProjectSlug: projectSlug, Status: "running"}, // same workflow -> DISTINCT + {ConnectionId: 1, WorkflowId: "wf-queued", Id: "job-3", ProjectSlug: projectSlug, Status: "queued"}, + {ConnectionId: 1, WorkflowId: "wf-success", Id: "job-4", ProjectSlug: projectSlug, Status: "success"}, // terminal -> excluded + {ConnectionId: 2, WorkflowId: "wf-otherconn", Id: "job-5", ProjectSlug: projectSlug, Status: "on_hold"}, // other connection -> excluded + } + for i := range seed { + assert.Nil(t, dataflowTester.Dal.Create(&seed[i])) + } + + cursor, err := dataflowTester.Dal.Cursor(tasks.UnfinishedJobsInputClauses(1, projectSlug)...) + assert.Nil(t, err) + iter, err := api.NewDalCursorIterator(dataflowTester.Dal, cursor, reflect.TypeOf(models.CircleciJob{})) + assert.Nil(t, err) + defer iter.Close() + + var ids []string + for iter.HasNext() { + item, err := iter.Fetch() + assert.Nil(t, err) + job := item.(*models.CircleciJob) + ids = append(ids, job.Id) + } + sort.Strings(ids) + + // Distinct workflow ids for connection 1's non-terminal jobs, with Id populated + // (the URL template reads .Input.Id). wf-success (terminal) and wf-otherconn + // (connection 2) are excluded. + assert.Equal(t, []string{"wf-onhold", "wf-queued"}, ids) + for _, id := range ids { + assert.NotEmpty(t, id, "Input.Id must be the workflow id, not empty (#8907)") + } +} diff --git a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv new file mode 100644 index 00000000000..24e3fbdcb6a --- /dev/null +++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_jobs_incremental.csv @@ -0,0 +1,73 @@ +id,params,data,url,input,created_at +20,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 5, + ""id"" : ""ab8c3282-0e74-4a41-834e-152a71280bed"", + ""started_at"" : ""2023-03-25T17:52:20Z"", + ""created_at"" : ""2023-03-25T17:52:20Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T17:52:23Z"" + }",https://circleci.com/api/v2/workflow/6731159f-5275-4bfa-ba70-39d343d63814/job,"{""id"": ""6731159f-5275-4bfa-ba70-39d343d63814"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""23622ee4-e150-4920-9d66-81533fa765a4"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:39:58.775 +21,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 7, + ""id"" : ""a00f80bc-f759-4900-97a5-2d121d80bde8"", + ""started_at"" : ""2023-03-25T17:56:27Z"", + ""created_at"" : ""2023-03-25T17:56:27Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T17:56:43Z"" + }",https://circleci.com/api/v2/workflow/7370985a-9de3-4a47-acbc-e6a1fe8e5812/job,"{""id"": ""7370985a-9de3-4a47-acbc-e6a1fe8e5812"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""866e967d-f826-4470-aed6-fc0c92e98703"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:39:58.953 +23,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 9, + ""id"" : ""2ff3594e-9da1-4306-aefa-77b72a97971e"", + ""started_at"" : ""2023-03-25T18:13:25Z"", + ""created_at"" : ""2023-03-25T18:13:25Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""success"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T18:13:38Z"" + }",https://circleci.com/api/v2/workflow/fd0bd4f5-264f-4e3c-a151-06153c018f78/job,"{""id"": ""fd0bd4f5-264f-4e3c-a151-06153c018f78"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""7fcc1623-edcc-4a76-ad20-cd81aa83519f"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:39:59.299 +24,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 6, + ""id"" : ""76c1f2cc-27ea-47aa-8167-48d2633abdba"", + ""started_at"" : ""2023-03-25T17:54:11Z"", + ""created_at"" : ""2023-03-25T17:54:11Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T17:54:23Z"" + }",https://circleci.com/api/v2/workflow/b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14/job,"{""id"": ""b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:39:59.482 +25,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 4, + ""id"" : ""a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b"", + ""started_at"" : ""2023-03-25T17:50:22Z"", + ""created_at"" : ""2023-03-25T17:50:22Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T17:50:25Z"" + }",https://circleci.com/api/v2/workflow/c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943/job,"{""id"": ""c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""2c45280f-7fb3-4025-b703-a547c4a94916"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:39:59.663 +27,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""dependencies"" : [ ], + ""job_number"" : 8, + ""id"" : ""004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e"", + ""started_at"" : ""2023-03-25T18:06:15Z"", + ""created_at"" : ""2023-03-25T18:06:15Z"", + ""name"" : ""build"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""type"" : ""build"", + ""stopped_at"" : ""2023-03-25T18:06:28Z"" + }",https://circleci.com/api/v2/workflow/fc76deef-bcdd-4856-8e96-a8e2d1c5a85f/job,"{""id"": ""fc76deef-bcdd-4856-8e96-a8e2d1c5a85f"", ""tag"": """", ""name"": """", ""status"": """", ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""errored_by"": """", ""started_by"": """", ""stopped_at"": null, ""canceled_by"": """", ""pipeline_id"": ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""ConnectionId"": 0, ""_raw_data_id"": 0, ""duration_sec"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""pipeline_number"": 0, ""_raw_data_params"": """", ""_raw_data_remark"": """"}",2023-03-28 15:40:00.022 diff --git a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv index 5760d52da61..a599e8aeb4a 100644 --- a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv +++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows.csv @@ -96,7 +96,7 @@ id,params,data,url,input,created_at ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", ""pipeline_number"" : 8, ""created_at"" : ""2023-03-25T18:06:13Z"", - ""stopped_at"" : ""2023-03-25T18:06:28Z"" + ""stopped_at"" : ""2023-03-25T18:05:28Z"" }",https://circleci.com/api/v2/pipeline/afe0cabe-e7ee-4eb7-bf13-bb6170d139f0/workflow,"{""id"": ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:58.245 28,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ ""pipeline_id"" : ""f5a2a6e8-4430-4935-8bec-77ca2b26c61d"", diff --git a/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv new file mode 100644 index 00000000000..2a70cdd511d --- /dev/null +++ b/backend/plugins/circleci/e2e/raw_tables/_raw_circleci_api_workflows_incremental.csv @@ -0,0 +1,67 @@ +id,params,data,url,input,created_at +19,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""23622ee4-e150-4920-9d66-81533fa765a4"", + ""id"" : ""6731159f-5275-4bfa-ba70-39d343d63814"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 5, + ""created_at"" : ""2023-03-25T17:52:15Z"", + ""stopped_at"" : ""2023-03-25T17:52:23Z"" + }",https://circleci.com/api/v2/pipeline/23622ee4-e150-4920-9d66-81533fa765a4/workflow,"{""id"": ""23622ee4-e150-4920-9d66-81533fa765a4"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:56.782 +20,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""2c45280f-7fb3-4025-b703-a547c4a94916"", + ""id"" : ""c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 4, + ""created_at"" : ""2023-03-25T17:50:20Z"", + ""stopped_at"" : ""2023-03-25T17:50:25Z"" + }",https://circleci.com/api/v2/pipeline/2c45280f-7fb3-4025-b703-a547c4a94916/workflow,"{""id"": ""2c45280f-7fb3-4025-b703-a547c4a94916"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:56.849 +22,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""7fcc1623-edcc-4a76-ad20-cd81aa83519f"", + ""id"" : ""fd0bd4f5-264f-4e3c-a151-06153c018f78"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""success"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 9, + ""created_at"" : ""2023-03-25T18:13:21Z"", + ""stopped_at"" : ""2023-03-25T18:13:38Z"" + }",https://circleci.com/api/v2/pipeline/7fcc1623-edcc-4a76-ad20-cd81aa83519f/workflow,"{""id"": ""7fcc1623-edcc-4a76-ad20-cd81aa83519f"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:57.179 +24,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""866e967d-f826-4470-aed6-fc0c92e98703"", + ""id"" : ""7370985a-9de3-4a47-acbc-e6a1fe8e5812"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 7, + ""created_at"" : ""2023-03-25T17:56:24Z"", + ""stopped_at"" : ""2023-03-25T17:56:43Z"" + }",https://circleci.com/api/v2/pipeline/866e967d-f826-4470-aed6-fc0c92e98703/workflow,"{""id"": ""866e967d-f826-4470-aed6-fc0c92e98703"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:57.524 +26,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"", + ""id"" : ""b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 6, + ""created_at"" : ""2023-03-25T17:54:09Z"", + ""stopped_at"" : ""2023-03-25T17:54:23Z"" + }",https://circleci.com/api/v2/pipeline/70f3eb15-3b94-4f80-b65e-f23f4b74c33a/workflow,"{""id"": ""70f3eb15-3b94-4f80-b65e-f23f4b74c33a"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:58.137 +27,"{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}","{ + ""pipeline_id"" : ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", + ""id"" : ""fc76deef-bcdd-4856-8e96-a8e2d1c5a85f"", + ""name"" : ""workflow"", + ""project_slug"" : ""gh/coldgust/coldgust.github.io"", + ""status"" : ""failed"", + ""started_by"" : ""1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7"", + ""pipeline_number"" : 8, + ""created_at"" : ""2023-03-25T18:06:13Z"", + ""stopped_at"" : ""2023-03-25T18:06:28Z"" + }",https://circleci.com/api/v2/pipeline/afe0cabe-e7ee-4eb7-bf13-bb6170d139f0/workflow,"{""id"": ""afe0cabe-e7ee-4eb7-bf13-bb6170d139f0"", ""vcs"": {""tag"": """", ""branch"": """", ""commit"": {""body"": """", ""subject"": """"}, ""revision"": """", ""review_id"": """", ""review_url"": """", ""provider_name"": """", ""origin_repository_url"": """", ""target_repository_url"": """"}, ""state"": """", ""errors"": null, ""number"": 0, ""trigger"": {""type"": """", ""actor"": {""login"": """", ""avatar_url"": """"}, ""received_at"": """"}, ""createdAt"": ""0001-01-01T00:00:00Z"", ""updatedAt"": ""0001-01-01T00:00:00Z"", ""created_at"": null, ""updated_at"": null, ""ConnectionId"": 0, ""_raw_data_id"": 0, ""project_slug"": """", ""_raw_data_table"": """", ""_raw_data_params"": """", ""_raw_data_remark"": """", ""trigger_parameters"": null}",2023-03-28 15:39:58.245 diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv index 82f2b4c45fc..517060bd3e0 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs.csv @@ -1,10 +1,10 @@ -connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration -1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0 -1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0 -1,89054eb2-8e85-4f5c-9a93-66d753a0e970,c46092f9-6f82-4a52-8d8b-bd70d365dfc2,github/coldgust/coldgust.github.io,,[],3,,2023-03-25T17:39:25.000+00:00,say-hello,,success,build,,2023-03-25T17:39:28.000+00:00,3,625ca634-68fe-4515-91f0-7ba8af51dc99,0 -1,8971a56b-5547-4824-94dd-07bb467524c5,7b96e45b-b10e-47a0-95d0-96580b88bdda,github/coldgust/coldgust.github.io,,[],2,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:23.000+00:00,3,87aad008-1ad5-486a-8174-fdeed846561a,0 -1,8fe60291-68f7-40e2-acec-d99bff4da713,afde48dd-7319-4973-b3c8-e00308ff7667,github/coldgust/coldgust.github.io,,[],1,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:22.000+00:00,2,afef32b3-5ffe-48d2-8d9e-46dcedd82554,0 -1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0 -1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0 -1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0 -1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0 +connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,created_date,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration +1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0 +1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0 +1,89054eb2-8e85-4f5c-9a93-66d753a0e970,c46092f9-6f82-4a52-8d8b-bd70d365dfc2,github/coldgust/coldgust.github.io,,[],3,2023-03-25T17:39:25.000+00:00,,2023-03-25T17:39:25.000+00:00,say-hello,,success,build,,2023-03-25T17:39:28.000+00:00,3,625ca634-68fe-4515-91f0-7ba8af51dc99,0 +1,8971a56b-5547-4824-94dd-07bb467524c5,7b96e45b-b10e-47a0-95d0-96580b88bdda,github/coldgust/coldgust.github.io,,[],2,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:23.000+00:00,3,87aad008-1ad5-486a-8174-fdeed846561a,0 +1,8fe60291-68f7-40e2-acec-d99bff4da713,afde48dd-7319-4973-b3c8-e00308ff7667,github/coldgust/coldgust.github.io,,[],1,2023-03-25T17:12:20.000+00:00,,2023-03-25T17:12:20.000+00:00,say-hello,,success,build,,2023-03-25T17:12:22.000+00:00,2,afef32b3-5ffe-48d2-8d9e-46dcedd82554,0 +1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0 +1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0 +1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0 +1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0 diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv new file mode 100644 index 00000000000..ea9a23a4456 --- /dev/null +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_jobs_incremental.csv @@ -0,0 +1,7 @@ +connection_id,workflow_id,id,project_slug,canceled_by,dependencies,job_number,created_date,queued_at,started_at,name,approved_by,status,type,approval_request_id,stopped_at,duration_sec,pipeline_id,duration +1,6731159f-5275-4bfa-ba70-39d343d63814,ab8c3282-0e74-4a41-834e-152a71280bed,github/coldgust/coldgust.github.io,,[],5,2023-03-25T17:52:20.000+00:00,,2023-03-25T17:52:20.000+00:00,build,,failed,build,,2023-03-25T17:52:23.000+00:00,3,23622ee4-e150-4920-9d66-81533fa765a4,0 +1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,a00f80bc-f759-4900-97a5-2d121d80bde8,github/coldgust/coldgust.github.io,,[],7,2023-03-25T17:56:27.000+00:00,,2023-03-25T17:56:27.000+00:00,build,,failed,build,,2023-03-25T17:56:43.000+00:00,16,866e967d-f826-4470-aed6-fc0c92e98703,0 +1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,76c1f2cc-27ea-47aa-8167-48d2633abdba,github/coldgust/coldgust.github.io,,[],6,2023-03-25T17:54:11.000+00:00,,2023-03-25T17:54:11.000+00:00,build,,failed,build,,2023-03-25T17:54:23.000+00:00,12,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,0 +1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,a4af3dd5-a3ae-48e8-b634-e2d63aafbb5b,github/coldgust/coldgust.github.io,,[],4,2023-03-25T17:50:22.000+00:00,,2023-03-25T17:50:22.000+00:00,build,,failed,build,,2023-03-25T17:50:25.000+00:00,3,2c45280f-7fb3-4025-b703-a547c4a94916,0 +1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,004e3e27-17d7-4ccb-9b21-a7f55bcf2b3e,github/coldgust/coldgust.github.io,,[],8,2023-03-25T18:06:15.000+00:00,,2023-03-25T18:06:15.000+00:00,build,,failed,build,,2023-03-25T18:06:28.000+00:00,13,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,0 +1,fd0bd4f5-264f-4e3c-a151-06153c018f78,2ff3594e-9da1-4306-aefa-77b72a97971e,github/coldgust/coldgust.github.io,,[],9,2023-03-25T18:13:25.000+00:00,,2023-03-25T18:13:25.000+00:00,build,,success,build,,2023-03-25T18:13:38.000+00:00,13,7fcc1623-edcc-4a76-ad20-cd81aa83519f,0 diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv index dd01015be55..3d55ea884ea 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_pipelines.csv @@ -1,11 +1,11 @@ -connection_id,id,project_slug,errors,updated_at,number,trigger_parameters,state,created_at,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25 17:52:15.651,5,,created,2023-03-25 17:52:15.651,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""d65e36c3316c6350c0eeb2dcaa6411a49d2aa29d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,26,"" -1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25 17:50:20.316,4,,created,2023-03-25 17:50:20.316,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""5d69b9233ed0579b0726caa9d36564cf90a7c63f"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,27,"" -1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25 17:39:23.233,3,,created,2023-03-25 17:39:23.233,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""578575831946b53b763ef2807580320f8d8ff12d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,28,"" -1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25 17:54:08.987,6,,created,2023-03-25 17:54:08.987,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""fd096b2dbc1ae2ea4c37e7af5101b123e7f60223"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,25,"" -1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25 18:13:21.590,9,,created,2023-03-25 18:13:21.590,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""49874a6ac988e2b3434501d4a4554c54583e6820"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,22,"" -1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25 17:56:24.409,7,,created,2023-03-25 17:56:24.409,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""ad3aab6479048a72b6aade91b36fdeb7125a16a6"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,24,"" -1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25 17:12:18.747,2,,created,2023-03-25 17:12:18.747,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,29,"" -1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25 18:06:13.404,8,,created,2023-03-25 18:06:13.404,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""7fb6961080f03defe9b7abb35a111a560e6ab1cc"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,23,"" -1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25 17:12:18.398,1,,created,2023-03-25 17:12:18.398,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,30,"" -1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed to fetch config.yml file.""}]",2023-03-25 18:13:43.446,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25 18:13:43.446,"{""type"":""Decoupled Ingestion System"",""received_at"":""1970-01-01T00:00:00.000Z"",""actor"":{""login"":"""",""avatar_url"":""""}}","{""provider_name"":"""",""target_repository_url"":"""",""branch"":"""",""review_id"":"""",""review_url"":"""",""revision"":"""",""tag"":"""",""origin_repository_url"":"""",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,21,"" +connection_id,id,project_slug,errors,updated_date,number,trigger_parameters,state,created_date,trigger,vcs,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,23622ee4-e150-4920-9d66-81533fa765a4,github/coldgust/coldgust.github.io,[],2023-03-25T17:52:15.651+00:00,5,,created,2023-03-25T17:52:15.651+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:52:15.446Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""d65e36c3316c6350c0eeb2dcaa6411a49d2aa29d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,26,"" +1,2c45280f-7fb3-4025-b703-a547c4a94916,github/coldgust/coldgust.github.io,[],2023-03-25T17:50:20.316+00:00,4,,created,2023-03-25T17:50:20.316+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:50:20.150Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""5d69b9233ed0579b0726caa9d36564cf90a7c63f"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,27,"" +1,625ca634-68fe-4515-91f0-7ba8af51dc99,github/coldgust/coldgust.github.io,[],2023-03-25T17:39:23.233+00:00,3,,created,2023-03-25T17:39:23.233+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:39:22.921Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""578575831946b53b763ef2807580320f8d8ff12d"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,28,"" +1,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,github/coldgust/coldgust.github.io,[],2023-03-25T17:54:08.987+00:00,6,,created,2023-03-25T17:54:08.987+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:54:08.742Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""fd096b2dbc1ae2ea4c37e7af5101b123e7f60223"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,25,"" +1,7fcc1623-edcc-4a76-ad20-cd81aa83519f,github/coldgust/coldgust.github.io,[],2023-03-25T18:13:21.590+00:00,9,,created,2023-03-25T18:13:21.590+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T18:13:21.404Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""49874a6ac988e2b3434501d4a4554c54583e6820"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,22,"" +1,866e967d-f826-4470-aed6-fc0c92e98703,github/coldgust/coldgust.github.io,[],2023-03-25T17:56:24.409+00:00,7,,created,2023-03-25T17:56:24.409+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:56:24.245Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""ad3aab6479048a72b6aade91b36fdeb7125a16a6"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,24,"" +1,87aad008-1ad5-486a-8174-fdeed846561a,github/coldgust/coldgust.github.io,[],2023-03-25T17:12:18.747+00:00,2,,created,2023-03-25T17:12:18.747+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T17:12:18.624Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""Add .circleci/config.yml"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,29,"" +1,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,github/coldgust/coldgust.github.io,[],2023-03-25T18:06:13.404+00:00,8,,created,2023-03-25T18:06:13.404+00:00,"{""type"":""webhook"",""received_at"":""2023-03-25T18:06:13.246Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""master"",""review_id"":"""",""review_url"":"""",""revision"":""7fb6961080f03defe9b7abb35a111a560e6ab1cc"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":""add circle"",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,23,"" +1,afef32b3-5ffe-48d2-8d9e-46dcedd82554,github/coldgust/coldgust.github.io,[],2023-03-25T17:12:18.398+00:00,1,,created,2023-03-25T17:12:18.398+00:00,"{""type"":""api"",""received_at"":""2023-03-25T17:12:16.438Z"",""actor"":{""login"":""coldgust"",""avatar_url"":""https://avatars.githubusercontent.com/u/115207009?v=4""}}","{""provider_name"":""GitHub"",""target_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""branch"":""circleci-project-setup"",""review_id"":"""",""review_url"":"""",""revision"":""9fcceda0e9749bc63e6397b23531fb6bf83634df"",""tag"":"""",""origin_repository_url"":""https://github.com/coldgust/coldgust.github.io"",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,30,"" +1,d323f088-02fa-4ed5-9696-fc2f89a27150,github/coldgust/coldgust.github.io,"[{""type"":""config-fetch"",""message"":""Failed to fetch config.yml file.""}]",2023-03-25T18:13:43.446+00:00,10,"{""circleci"":{""org_name"":""coldgust"",""project_name"":""coldgust.github.io"",""provider_name"":""github""},""git"":{""checkout_sha"":""6ba0cdaf7e7791c5bc5aa89a13b9cd184f8ad296""}}",errored,2023-03-25T18:13:43.446+00:00,"{""type"":""Decoupled Ingestion System"",""received_at"":""1970-01-01T00:00:00.000Z"",""actor"":{""login"":"""",""avatar_url"":""""}}","{""provider_name"":"""",""target_repository_url"":"""",""branch"":"""",""review_id"":"""",""review_url"":"""",""revision"":"""",""tag"":"""",""origin_repository_url"":"""",""commit"":{""subject"":"""",""body"":""""}}","{""ConnectionId"":1,""ProjectSlug"":""github/coldgust/coldgust.github.io""}",_raw_circleci_api_pipelines,21,"" diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv index bd6af7cc904..eb1e03e8e11 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows.csv @@ -7,5 +7,5 @@ connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status 1,b3b77371-e27a-4e93-a554-c728f757cd24,github/coldgust/coldgust.github.io,f5a2a6e8-4430-4935-8bec-77ca2b26c61d,,workflow,,,failed,,16,,,0 1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14 1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5 -1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15 +1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:05:28.000+00:00,0 1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17 diff --git a/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv new file mode 100644 index 00000000000..529e847441b --- /dev/null +++ b/backend/plugins/circleci/e2e/snapshot_tables/_tool_circleci_workflows_incremental.csv @@ -0,0 +1,7 @@ +connection_id,id,project_slug,pipeline_id,canceled_by,name,errored_by,tag,status,started_by,pipeline_number,created_date,stopped_date,duration_sec +1,6731159f-5275-4bfa-ba70-39d343d63814,github/coldgust/coldgust.github.io,23622ee4-e150-4920-9d66-81533fa765a4,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,5,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:23.000+00:00,8 +1,7370985a-9de3-4a47-acbc-e6a1fe8e5812,github/coldgust/coldgust.github.io,866e967d-f826-4470-aed6-fc0c92e98703,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,7,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:43.000+00:00,19 +1,b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,github/coldgust/coldgust.github.io,70f3eb15-3b94-4f80-b65e-f23f4b74c33a,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,6,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:23.000+00:00,14 +1,c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,github/coldgust/coldgust.github.io,2c45280f-7fb3-4025-b703-a547c4a94916,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,4,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:25.000+00:00,5 +1,fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,github/coldgust/coldgust.github.io,afe0cabe-e7ee-4eb7-bf13-bb6170d139f0,,workflow,,,failed,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,8,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:28.000+00:00,15 +1,fd0bd4f5-264f-4e3c-a151-06153c018f78,github/coldgust/coldgust.github.io,7fcc1623-edcc-4a76-ad20-cd81aa83519f,,workflow,,,success,1c762fc2-b0fb-4fe2-97d2-5e54ddd1eba7,9,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:38.000+00:00,17 diff --git a/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv b/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv index 45f61d9cea6..40e7b614a9d 100644 --- a/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv +++ b/backend/plugins/circleci/e2e/snapshot_tables/cicd_pipelines.csv @@ -1,10 +1,10 @@ -id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,queued_date,started_date,cicd_scope_id -circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,,2023-03-25T17:52:15.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,,2023-03-25T17:56:24.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:39:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,,2023-03-25T17:54:09.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,,2023-03-25T17:50:20.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,15,,PRODUCTION,,2023-03-25T18:06:13.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io -circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,,2023-03-25T18:13:21.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +id,name,display_title,url,result,status,original_status,original_result,type,duration_sec,queued_duration_sec,environment,queued_date,started_date,created_date,cicd_scope_id +circleci:CircleciWorkflow:1:6731159f-5275-4bfa-ba70-39d343d63814,workflow,workflow#5,,FAILURE,DONE,failed,,,8,,PRODUCTION,,2023-03-25T17:52:15.000+00:00,2023-03-25T17:52:15.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:7370985a-9de3-4a47-acbc-e6a1fe8e5812,workflow,workflow#7,,FAILURE,DONE,failed,,,19,,PRODUCTION,,2023-03-25T17:56:24.000+00:00,2023-03-25T17:56:24.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:89054eb2-8e85-4f5c-9a93-66d753a0e970,say-hello-workflow,say-hello-workflow#3,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:39:23.000+00:00,2023-03-25T17:39:23.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:8971a56b-5547-4824-94dd-07bb467524c5,say-hello-workflow,say-hello-workflow#2,,SUCCESS,DONE,success,,,5,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:8fe60291-68f7-40e2-acec-d99bff4da713,say-hello-workflow,say-hello-workflow#1,,SUCCESS,DONE,success,,,4,,PRODUCTION,,2023-03-25T17:12:18.000+00:00,2023-03-25T17:12:18.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14,workflow,workflow#6,,FAILURE,DONE,failed,,,14,,PRODUCTION,,2023-03-25T17:54:09.000+00:00,2023-03-25T17:54:09.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943,workflow,workflow#4,,FAILURE,DONE,failed,,,5,,PRODUCTION,,2023-03-25T17:50:20.000+00:00,2023-03-25T17:50:20.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:fc76deef-bcdd-4856-8e96-a8e2d1c5a85f,workflow,workflow#8,,FAILURE,DONE,failed,,,0,,PRODUCTION,,2023-03-25T18:06:13.000+00:00,2023-03-25T18:06:13.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io +circleci:CircleciWorkflow:1:fd0bd4f5-264f-4e3c-a151-06153c018f78,workflow,workflow#9,,SUCCESS,DONE,success,,,17,,PRODUCTION,,2023-03-25T18:13:21.000+00:00,2023-03-25T18:13:21.000+00:00,circleci:CircleciProject:1:github/coldgust/coldgust.github.io diff --git a/backend/plugins/circleci/e2e/workflow_test.go b/backend/plugins/circleci/e2e/workflow_test.go index cc542fdb27f..ba5194cea68 100644 --- a/backend/plugins/circleci/e2e/workflow_test.go +++ b/backend/plugins/circleci/e2e/workflow_test.go @@ -72,7 +72,7 @@ func TestCircleciWorkflow(t *testing.T) { devops.CICDPipeline{}, e2ehelper.TableOptions{ CSVRelPath: "./snapshot_tables/cicd_pipelines.csv", - IgnoreFields: []string{"finished_date", "created_date", "is_child"}, + IgnoreFields: []string{"finished_date", "is_child"}, IgnoreTypes: []interface{}{domainlayer.DomainEntity{}}, }, ) diff --git a/backend/plugins/circleci/models/job.go b/backend/plugins/circleci/models/job.go index bafaf5b9ebb..c8a3bea31cb 100644 --- a/backend/plugins/circleci/models/job.go +++ b/backend/plugins/circleci/models/job.go @@ -29,7 +29,7 @@ type CircleciJob struct { CanceledBy string `gorm:"type:varchar(100)" json:"canceled_by"` Dependencies []string `gorm:"serializer:json;type:text" json:"dependencies"` JobNumber int64 `json:"job_number"` - CreatedAt *common.Iso8601Time `json:"created_at"` + CreatedDate *common.Iso8601Time `json:"created_at"` QueuedAt *common.Iso8601Time `json:"queued_at"` StartedAt *common.Iso8601Time `json:"started_at"` Name string `gorm:"type:varchar(255)" json:"name"` diff --git a/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go b/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go new file mode 100644 index 00000000000..497c0d3b5a6 --- /dev/null +++ b/backend/plugins/circleci/models/migrationscripts/20260322_rename_created_at_to_created_date.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +type circleciWorkflow20260322 struct { + CreatedDate *time.Time +} + +func (circleciWorkflow20260322) TableName() string { + return "_tool_circleci_workflows" +} + +type circleciJob20260322 struct { + CreatedDate *time.Time +} + +func (circleciJob20260322) TableName() string { + return "_tool_circleci_jobs" +} + +type renameCreatedAtToCreatedDate20260322 struct{} + +func (*renameCreatedAtToCreatedDate20260322) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + if err := db.AutoMigrate(&circleciWorkflow20260322{}); err != nil { + return err + } + if err := db.Exec("UPDATE _tool_circleci_workflows SET created_date = created_at WHERE created_date IS NULL"); err != nil { + return err + } + + if err := db.AutoMigrate(&circleciJob20260322{}); err != nil { + return err + } + if err := db.Exec("UPDATE _tool_circleci_jobs SET created_date = created_at WHERE created_date IS NULL"); err != nil { + return err + } + + return nil +} + +func (*renameCreatedAtToCreatedDate20260322) Version() uint64 { + return 20260322000001 +} + +func (*renameCreatedAtToCreatedDate20260322) Name() string { + return "circleci rename created_at to created_date in workflows and jobs" +} diff --git a/backend/plugins/circleci/models/migrationscripts/register.go b/backend/plugins/circleci/models/migrationscripts/register.go index 5a23f27cc85..af8472a3cd7 100644 --- a/backend/plugins/circleci/models/migrationscripts/register.go +++ b/backend/plugins/circleci/models/migrationscripts/register.go @@ -24,5 +24,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(addFieldsToCircleciJob20231129), + new(renameCreatedAtToCreatedDate20260322), } } diff --git a/backend/plugins/circleci/models/pipeline.go b/backend/plugins/circleci/models/pipeline.go index a18cded109d..a775ac4b2f4 100644 --- a/backend/plugins/circleci/models/pipeline.go +++ b/backend/plugins/circleci/models/pipeline.go @@ -49,8 +49,8 @@ type CircleciPipeline struct { ConnectionId uint64 `gorm:"primaryKey;type:BIGINT" json:"connectionId" mapstructure:"connectionId"` Id string `gorm:"primaryKey;type:varchar(100)" json:"id" mapstructure:"id"` ProjectSlug string `gorm:"type:varchar(255)" json:"projectSlug" mapstructure:"projectSlug"` - UpdatedDate *common.Iso8601Time `json:"updatedDate" mapstructure:"updatedDate"` - CreatedDate *common.Iso8601Time `json:"createdDate" mapstructure:"createdDate"` + UpdatedDate *common.Iso8601Time `json:"updated_at"` + CreatedDate *common.Iso8601Time `json:"created_at"` Number int64 `json:"number" mapstructure:"number"` // pipeline number within the project? TriggerParameters any `gorm:"serializer:json" json:"trigger_parameters" mapstructure:"triggerParameters"` State string `gorm:"type:varchar(100)" json:"state" mapstructure:"state"` diff --git a/backend/plugins/circleci/models/workflow.go b/backend/plugins/circleci/models/workflow.go index f4e5e9f7111..0b1a330284e 100644 --- a/backend/plugins/circleci/models/workflow.go +++ b/backend/plugins/circleci/models/workflow.go @@ -33,7 +33,7 @@ type CircleciWorkflow struct { Status string `gorm:"type:varchar(100)" json:"status"` StartedBy string `gorm:"type:varchar(100)" json:"started_by"` PipelineNumber int64 `json:"pipeline_number"` - CreatedAt *common.Iso8601Time `json:"created_at"` + CreatedDate *common.Iso8601Time `json:"created_at"` StoppedAt *common.Iso8601Time `json:"stopped_at"` DurationSec float64 `json:"duration_sec"` diff --git a/backend/plugins/circleci/tasks/job_collector.go b/backend/plugins/circleci/tasks/job_collector.go index fd1d78286bf..ca158c1b934 100644 --- a/backend/plugins/circleci/tasks/job_collector.go +++ b/backend/plugins/circleci/tasks/job_collector.go @@ -41,6 +41,20 @@ var CollectJobsMeta = plugin.SubTaskMeta{ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, } +// UnfinishedJobsInputClauses returns the DAL clauses that select the workflows whose +// jobs are still in a non-terminal status and therefore need their job details +// recollected by the CollectJobs "unfinished details" collector. +func UnfinishedJobsInputClauses(connectionId uint64, projectSlug string) []dal.Clause { + return []dal.Clause{ + dal.Select("DISTINCT workflow_id AS id"), // #8907: alias to id so {{ .Input.Id }} resolves when scanned into CircleciJob + dal.From(&models.CircleciJob{}), + dal.Where( + "connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", + connectionId, projectSlug, + ), + } +} + func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) logger := taskCtx.GetLogger() @@ -59,8 +73,10 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { dal.Where("connection_id = ? and project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug), } - if isIncremental { - clauses = append(clauses, dal.Where("created_date > ?", createdAfter)) + // Incremental: workflows newer than last successful collectJobs. + // Full sync: workflows within SyncPolicy.TimeAfter window. + if createdAfter != nil { + clauses = append(clauses, dal.Where("created_date >= ?", createdAfter)) } db := taskCtx.GetDal() @@ -74,7 +90,7 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "/v2/workflow/{{ .Input.Id }}/job", Query: BuildQueryParamsWithPageToken, ResponseParser: ParseCircleciPageTokenResp, - AfterResponse: ignoreDeletedBuilds, // Ignore the 404 response if a workflow has been deleted + AfterResponse: ignoreDeletedOrBrokenBuilds, }, GetCreated: func(item json.RawMessage) (time.Time, errors.Error) { var job struct { // Individual job response lacks created_at field, so have to use started_at @@ -91,17 +107,11 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "/v2/workflow/{{ .Input.Id }}/job", // The individual job endpoint has different fields so need to recollect all jobs for a workflow Query: BuildQueryParamsWithPageToken, ResponseParser: ParseCircleciPageTokenResp, - AfterResponse: ignoreDeletedBuilds, + AfterResponse: ignoreDeletedOrBrokenBuilds, }, BuildInputIterator: func() (api.Iterator, errors.Error) { - clauses := []dal.Clause{ - dal.Select("DISTINCT workflow_id"), // Only need to recollect jobs for a workflow once - dal.From(&models.CircleciJob{}), - dal.Where("connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", data.Options.ConnectionId, data.Options.ProjectSlug), - } - db := taskCtx.GetDal() - cursor, err := db.Cursor(clauses...) + cursor, err := db.Cursor(UnfinishedJobsInputClauses(data.Options.ConnectionId, data.Options.ProjectSlug)...) if err != nil { return nil, err } diff --git a/backend/plugins/circleci/tasks/job_converter.go b/backend/plugins/circleci/tasks/job_converter.go index 2cb8de1207c..5a085609d9f 100644 --- a/backend/plugins/circleci/tasks/job_converter.go +++ b/backend/plugins/circleci/tasks/job_converter.go @@ -58,8 +58,8 @@ func ConvertJobs(taskCtx plugin.SubTaskContext) errors.Error { Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciJob) createdAt := time.Now() - if userTool.CreatedAt != nil { - createdAt = userTool.CreatedAt.ToTime() + if userTool.CreatedDate != nil { + createdAt = userTool.CreatedDate.ToTime() } task := &devops.CICDTask{ DomainEntity: domainlayer.DomainEntity{ diff --git a/backend/plugins/circleci/tasks/shared.go b/backend/plugins/circleci/tasks/shared.go index 6e235ecd6b0..d6bbee8120f 100644 --- a/backend/plugins/circleci/tasks/shared.go +++ b/backend/plugins/circleci/tasks/shared.go @@ -123,10 +123,15 @@ func ParseCircleciPageTokenResp(res *http.Response) ([]json.RawMessage, errors.E return data.Items, err } -func ignoreDeletedBuilds(res *http.Response) errors.Error { - // CircleCI API will return a 404 response for a workflow/job that has been deleted - // due to their data retention policy. We should ignore these errors. - if res.StatusCode == http.StatusNotFound { +// ignoreDeletedOrBrokenBuilds skips per-item API failures that should not +// abort an entire collector subtask. 404 means the resource was deleted +// (retention); 500 means the CircleCI Server record is corrupt/stuck +// (e.g. pipeline exists but its /workflow endpoint errors). +func ignoreDeletedOrBrokenBuilds(res *http.Response) errors.Error { + switch res.StatusCode { + case http.StatusNotFound: + return api.ErrIgnoreAndContinue + case http.StatusInternalServerError: return api.ErrIgnoreAndContinue } return nil diff --git a/backend/plugins/circleci/tasks/shared_test.go b/backend/plugins/circleci/tasks/shared_test.go new file mode 100644 index 00000000000..a6c55e9c9c7 --- /dev/null +++ b/backend/plugins/circleci/tasks/shared_test.go @@ -0,0 +1,62 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func makeResponse(statusCode int) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(bytes.NewBufferString("")), + Request: &http.Request{}, + } +} + +func TestIgnoreDeletedOrBrokenBuilds(t *testing.T) { + tests := []struct { + name string + statusCode int + want error + }{ + {"404 returns ErrIgnoreAndContinue", http.StatusNotFound, api.ErrIgnoreAndContinue}, + {"500 returns ErrIgnoreAndContinue", http.StatusInternalServerError, api.ErrIgnoreAndContinue}, + {"200 returns nil", http.StatusOK, nil}, + {"403 returns nil", http.StatusForbidden, nil}, + {"502 returns nil", http.StatusBadGateway, nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + res := makeResponse(tt.statusCode) + got := ignoreDeletedOrBrokenBuilds(res) + if tt.want == nil { + assert.Nil(t, got) + } else { + assert.Equal(t, tt.want, got) + } + }) + } +} + diff --git a/backend/plugins/circleci/tasks/workflow_collector.go b/backend/plugins/circleci/tasks/workflow_collector.go index f0f3aebe5c9..25c5c727d36 100644 --- a/backend/plugins/circleci/tasks/workflow_collector.go +++ b/backend/plugins/circleci/tasks/workflow_collector.go @@ -60,8 +60,10 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error { dal.Where("connection_id = ? AND project_slug = ?", data.Options.ConnectionId, data.Options.ProjectSlug), } - if isIncremental { - clauses = append(clauses, dal.Where("created_date > ?", createdAfter)) + // Incremental: pipelines newer than last successful collectWorkflows. + // Full sync: pipelines within SyncPolicy.TimeAfter window. + if createdAfter != nil { + clauses = append(clauses, dal.Where("created_date >= ?", createdAfter)) } db := taskCtx.GetDal() @@ -75,7 +77,7 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "/v2/pipeline/{{ .Input.Id }}/workflow", Query: BuildQueryParamsWithPageToken, ResponseParser: ParseCircleciPageTokenResp, - AfterResponse: ignoreDeletedBuilds, // Ignore the 404 response if a workflow has been deleted + AfterResponse: ignoreDeletedOrBrokenBuilds, }, GetCreated: extractCreatedAt, }, @@ -88,7 +90,7 @@ func CollectWorkflows(taskCtx plugin.SubTaskContext) errors.Error { err := api.UnmarshalResponse(res, &data) return []json.RawMessage{data}, err }, - AfterResponse: ignoreDeletedBuilds, + AfterResponse: ignoreDeletedOrBrokenBuilds, }, BuildInputIterator: func() (api.Iterator, errors.Error) { clauses := []dal.Clause{ diff --git a/backend/plugins/circleci/tasks/workflow_converter.go b/backend/plugins/circleci/tasks/workflow_converter.go index a023accc772..bf6b029e3e5 100644 --- a/backend/plugins/circleci/tasks/workflow_converter.go +++ b/backend/plugins/circleci/tasks/workflow_converter.go @@ -61,11 +61,11 @@ func ConvertWorkflows(taskCtx plugin.SubTaskContext) errors.Error { Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { userTool := inputRow.(*models.CircleciWorkflow) // Skip if CreatedAt is null or empty string - still enters into the `_tool_circleci_workflows` table with null values - if userTool.CreatedAt.ToNullableTime() == nil { - logger.Info("CreatedAt is null or empty string in the CircleCI API response for %s", userTool.PipelineId) + if userTool.CreatedDate.ToNullableTime() == nil { + logger.Info("CreatedDate is null or empty string in the CircleCI API response for %s", userTool.PipelineId) return []interface{}{}, nil } - createdAt := userTool.CreatedAt.ToTime() + createdAt := userTool.CreatedDate.ToTime() pipeline := &devops.CICDPipeline{ DomainEntity: domainlayer.DomainEntity{ Id: getWorkflowIdGen().Generate(data.Options.ConnectionId, userTool.Id), diff --git a/backend/plugins/circleci/tasks/workflow_extractor.go b/backend/plugins/circleci/tasks/workflow_extractor.go index ba0639bc877..27ec520a07c 100644 --- a/backend/plugins/circleci/tasks/workflow_extractor.go +++ b/backend/plugins/circleci/tasks/workflow_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" @@ -48,10 +49,13 @@ func ExtractWorkflows(taskCtx plugin.SubTaskContext) errors.Error { toolL := userRes toolL.ConnectionId = data.Options.ConnectionId toolL.ProjectSlug = data.Options.ProjectSlug - if userRes.CreatedAt != nil && userRes.StoppedAt != nil { - startTime := userRes.CreatedAt.ToTime() + if userRes.CreatedDate != nil && userRes.StoppedAt != nil { + startTime := userRes.CreatedDate.ToTime() endTime := userRes.StoppedAt.ToTime() - toolL.DurationSec = float64(endTime.Sub(startTime).Milliseconds() / 1e3) + durationSec := endTime.Sub(startTime).Seconds() + if durationSec >= 0 { + toolL.DurationSec = durationSec + } } return []interface{}{ &toolL, diff --git a/backend/plugins/claude_code/api/blueprint_v200.go b/backend/plugins/claude_code/api/blueprint_v200.go new file mode 100644 index 00000000000..3802b70a0e2 --- /dev/null +++ b/backend/plugins/claude_code/api/blueprint_v200.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/claude_code/models" + "github.com/apache/incubator-devlake/plugins/claude_code/tasks" +) + +// MakeDataSourcePipelinePlanV200 generates the pipeline plan for blueprint v2.0.0. +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + _, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails) + if err != nil { + return nil, nil, err + } + + return plan, nil, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.ClaudeCodeScope, models.ClaudeCodeScopeConfig], +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + task, err := helper.MakePipelinePlanTask( + "claude_code", + subtaskMetas, + nil, + tasks.ClaudeCodeOptions{ + ConnectionId: scope.ConnectionId, + ScopeId: scope.Id, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} diff --git a/backend/plugins/claude_code/api/connection.go b/backend/plugins/claude_code/api/connection.go new file mode 100644 index 00000000000..2cfc6d2fb04 --- /dev/null +++ b/backend/plugins/claude_code/api/connection.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// PostConnections creates a new Claude Code connection. +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, err + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.ClaudeCodeConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.ClaudeCodeConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.ClaudeCodeConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func validateConnection(connection *models.ClaudeCodeConnection) errors.Error { + if connection == nil { + return errors.BadInput.New("connection is required") + } + hasToken := strings.TrimSpace(connection.Token) != "" + if connection.HasIncompleteCustomHeaders() { + return errors.BadInput.New("custom headers must include both key and value") + } + hasCustomHeaders := connection.HasUsableCustomHeaders() + if !hasToken && !hasCustomHeaders { + return errors.BadInput.New("either token or at least one custom header is required") + } + if strings.TrimSpace(connection.Organization) == "" { + return errors.BadInput.New("organization is required") + } + if connection.RateLimitPerHour < 0 { + return errors.BadInput.New("rateLimitPerHour must be non-negative") + } + return nil +} diff --git a/backend/plugins/claude_code/api/connection_test.go b/backend/plugins/claude_code/api/connection_test.go new file mode 100644 index 00000000000..efa81d42149 --- /dev/null +++ b/backend/plugins/claude_code/api/connection_test.go @@ -0,0 +1,135 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +const ( + testOrganization = "anthropic-labs" + testToken = "sk-ant-example" +) + +func TestValidateConnectionSuccess(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + Token: testToken, + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.NoError(t, err) +} + +func TestValidateConnectionNil(t *testing.T) { + err := validateConnection(nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "connection is required") +} + +func TestValidateConnectionMissingOrganization(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Token: testToken, + }, + } + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "organization is required") +} + +func TestValidateConnectionMissingToken(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + }, + } + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "either token or at least one custom header is required") +} + +func TestValidateConnectionCustomHeadersWithoutToken(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + CustomHeaders: []models.CustomHeader{ + {Key: "Ocp-Apim-Subscription-Key", Value: "secret-key"}, + }, + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.NoError(t, err) +} + +func TestValidateConnectionRejectsBlankCustomHeaders(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + CustomHeaders: []models.CustomHeader{ + {Key: "", Value: ""}, + }, + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "either token or at least one custom header is required") +} + +func TestValidateConnectionRejectsIncompleteCustomHeaders(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + CustomHeaders: []models.CustomHeader{ + {Key: "Ocp-Apim-Subscription-Key", Value: ""}, + }, + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "custom headers must include both key and value") +} + +func TestValidateConnectionInvalidRateLimit(t *testing.T) { + connection := &models.ClaudeCodeConnection{ + ClaudeCodeConn: models.ClaudeCodeConn{ + Organization: testOrganization, + Token: testToken, + }, + } + connection.RateLimitPerHour = -1 + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "rateLimitPerHour must be non-negative") +} diff --git a/backend/plugins/claude_code/api/init.go b/backend/plugins/claude_code/api/init.go new file mode 100644 index 00000000000..3b27e4a5636 --- /dev/null +++ b/backend/plugins/claude_code/api/init.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +var ( + basicRes context.BasicRes + vld *validator.Validate + connectionHelper *helper.ConnectionApiHelper + dsHelper *helper.DsHelper[models.ClaudeCodeConnection, models.ClaudeCodeScope, models.ClaudeCodeScopeConfig] + raProxy *helper.DsRemoteApiProxyHelper[models.ClaudeCodeConnection] + raScopeList *helper.DsRemoteApiScopeListHelper[models.ClaudeCodeConnection, models.ClaudeCodeScope, ClaudeCodeRemotePagination] +) + +// Init stores basic resources and configures shared helpers for API handlers. +func Init(br context.BasicRes, meta plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = helper.NewConnectionHelper(basicRes, vld, meta.Name()) + dsHelper = helper.NewDataSourceHelper[ + models.ClaudeCodeConnection, models.ClaudeCodeScope, models.ClaudeCodeScopeConfig, + ]( + basicRes, + meta.Name(), + []string{"id", "organizationId"}, + func(c models.ClaudeCodeConnection) models.ClaudeCodeConnection { + c.Normalize() + return c.Sanitize() + }, + func(s models.ClaudeCodeScope) models.ClaudeCodeScope { return s }, + nil, + ) + raProxy = helper.NewDsRemoteApiProxyHelper[models.ClaudeCodeConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = helper.NewDsRemoteApiScopeListHelper[models.ClaudeCodeConnection, models.ClaudeCodeScope, ClaudeCodeRemotePagination](raProxy, listClaudeCodeRemoteScopes) +} diff --git a/backend/plugins/claude_code/api/remote_api.go b/backend/plugins/claude_code/api/remote_api.go new file mode 100644 index 00000000000..9c2e53ec112 --- /dev/null +++ b/backend/plugins/claude_code/api/remote_api.go @@ -0,0 +1,113 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helperapi "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/helpers/utils" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// ClaudeCodeRemotePagination is a placeholder for remote scope pagination. +// Claude Code currently returns a single organization scope per connection. +type ClaudeCodeRemotePagination struct { + Page int `json:"page"` +} + +func listClaudeCodeRemoteScopes( + connection *models.ClaudeCodeConnection, + _ plugin.ApiClient, + _ string, + _ ClaudeCodeRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.ClaudeCodeScope], + nextPage *ClaudeCodeRemotePagination, + err errors.Error, +) { + if connection == nil { + return nil, nil, errors.BadInput.New("connection is required") + } + + organizationId := strings.TrimSpace(connection.Organization) + if organizationId == "" { + return []dsmodels.DsRemoteApiScopeListEntry[models.ClaudeCodeScope]{}, nil, nil + } + + children = append(children, makeClaudeCodeRemoteScopeEntry(organizationId)) + return children, nil, nil +} + +func makeClaudeCodeRemoteScopeEntry(organizationId string) dsmodels.DsRemoteApiScopeListEntry[models.ClaudeCodeScope] { + organizationId = strings.TrimSpace(organizationId) + return dsmodels.DsRemoteApiScopeListEntry[models.ClaudeCodeScope]{ + Type: helperapi.RAS_ENTRY_TYPE_SCOPE, + Id: organizationId, + Name: organizationId, + FullName: organizationId, + Data: &models.ClaudeCodeScope{ + Id: organizationId, + Organization: organizationId, + Name: organizationId, + FullName: organizationId, + }, + } +} + +// RemoteScopes lists all available scopes for this connection. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// SearchRemoteScopes searches scopes for this connection. +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + params := &dsmodels.DsRemoteApiScopeSearchParams{ + Page: 1, + PageSize: 50, + } + if err := utils.DecodeMapStruct(input.Query, params, true); err != nil { + return nil, err + } + if err := errors.Convert(vld.Struct(params)); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid params") + } + + children := []dsmodels.DsRemoteApiScopeListEntry[models.ClaudeCodeScope]{} + organizationId := strings.TrimSpace(connection.Organization) + searchLower := strings.ToLower(strings.TrimSpace(params.Search)) + if organizationId != "" && strings.Contains(strings.ToLower(organizationId), searchLower) { + children = append(children, makeClaudeCodeRemoteScopeEntry(organizationId)) + } + + return &plugin.ApiResourceOutput{ + Body: map[string]interface{}{ + "children": children, + "page": params.Page, + "pageSize": params.PageSize, + }, + }, nil +} diff --git a/backend/plugins/claude_code/api/scope.go b/backend/plugins/claude_code/api/scope.go new file mode 100644 index 00000000000..ce8dea71797 --- /dev/null +++ b/backend/plugins/claude_code/api/scope.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// GetScopeList returns all scopes for a connection. +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// PutScopes creates or updates scopes for a connection. +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScope returns a single scope by id. +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope partially updates a scope. +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope. +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState returns the latest sync state for a scope. +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/claude_code/api/scope_config.go b/backend/plugins/claude_code/api/scope_config.go new file mode 100644 index 00000000000..571e49e85ca --- /dev/null +++ b/backend/plugins/claude_code/api/scope_config.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig creates a new scope config. +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// GetScopeConfigList returns all scope configs for a connection. +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// GetScopeConfig returns a single scope config by id. +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// PatchScopeConfig partially updates a scope config. +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// DeleteScopeConfig removes a scope config. +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig returns projects associated with a scope config. +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/claude_code/api/test_connection.go b/backend/plugins/claude_code/api/test_connection.go new file mode 100644 index 00000000000..45a75d45460 --- /dev/null +++ b/backend/plugins/claude_code/api/test_connection.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + gocontext "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" + "github.com/apache/incubator-devlake/plugins/claude_code/service" +) + +// TestConnection validates a Claude Code connection before saving it. +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(gocontext.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection validates a stored Claude Code connection with optional overrides. +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.ClaudeCodeConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, errors.BadInput.Wrap(err, "find connection from db")) + } + if err := (&models.ClaudeCodeConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, errors.Convert(err)) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(gocontext.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/claude_code/claude_code.go b/backend/plugins/claude_code/claude_code.go new file mode 100644 index 00000000000..73fedee29ee --- /dev/null +++ b/backend/plugins/claude_code/claude_code.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/claude_code/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.ClaudeCode + +// standalone mode for debugging collectors. +func main() { + cmd := &cobra.Command{Use: "claude_code"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "claude_code connection id") + scopeId := cmd.Flags().StringP("scopeId", "s", "", "claude_code scope id (organization)") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data created after the specified time") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("scopeId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "scopeId": *scopeId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/claude_code/impl/connection_helper.go b/backend/plugins/claude_code/impl/connection_helper.go new file mode 100644 index 00000000000..8b058d5344e --- /dev/null +++ b/backend/plugins/claude_code/impl/connection_helper.go @@ -0,0 +1,25 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import "github.com/apache/incubator-devlake/plugins/claude_code/models" + +// NormalizeConnection ensures required defaults are set before use. +func NormalizeConnection(connection *models.ClaudeCodeConnection) { + connection.Normalize() +} diff --git a/backend/plugins/claude_code/impl/impl.go b/backend/plugins/claude_code/impl/impl.go new file mode 100644 index 00000000000..4e7d2b136d1 --- /dev/null +++ b/backend/plugins/claude_code/impl/impl.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" + "github.com/apache/incubator-devlake/plugins/claude_code/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/claude_code/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.DataSourcePluginBlueprintV200 + plugin.PluginMigration + plugin.CloseablePluginTask +} = (*ClaudeCode)(nil) + +// ClaudeCode is the plugin entrypoint implementing DevLake interfaces. +type ClaudeCode struct{} + +func (p ClaudeCode) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p ClaudeCode) Description() string { + return "Collect Claude Code usage analytics and productivity metrics" +} + +func (p ClaudeCode) Name() string { + return "claude_code" +} + +func (p ClaudeCode) Connection() dal.Tabler { + return &models.ClaudeCodeConnection{} +} + +func (p ClaudeCode) Scope() plugin.ToolLayerScope { + return &models.ClaudeCodeScope{} +} + +func (p ClaudeCode) ScopeConfig() dal.Tabler { + return &models.ClaudeCodeScopeConfig{} +} + +func (p ClaudeCode) GetTablesInfo() []dal.Tabler { + return models.GetTablesInfo() +} + +func (p ClaudeCode) SubTaskMetas() []plugin.SubTaskMeta { + return tasks.GetSubTaskMetas() +} + +func (p ClaudeCode) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.ClaudeCodeOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.ClaudeCodeConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + NormalizeConnection(connection) + + taskData := &tasks.ClaudeCodeTaskData{ + Options: &op, + Connection: connection, + } + + return taskData, nil +} + +func (p ClaudeCode) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "GET": api.GetScopeConfig, + "PATCH": api.PatchScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p ClaudeCode) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p ClaudeCode) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/claude_code" +} + +func (p ClaudeCode) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p ClaudeCode) Close(taskCtx plugin.TaskContext) errors.Error { + return nil +} diff --git a/backend/plugins/claude_code/models/activity_summary.go b/backend/plugins/claude_code/models/activity_summary.go new file mode 100644 index 00000000000..3e81f6b0676 --- /dev/null +++ b/backend/plugins/claude_code/models/activity_summary.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClaudeCodeActivitySummary captures daily organisation-level engagement and seat +// utilisation from the /v1/organizations/analytics/summaries endpoint. +type ClaudeCodeActivitySummary struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` // = starting_date + + DailyActiveUserCount int `json:"dailyActiveUserCount"` + WeeklyActiveUserCount int `json:"weeklyActiveUserCount"` + MonthlyActiveUserCount int `json:"monthlyActiveUserCount"` + AssignedSeatCount int `json:"assignedSeatCount"` + PendingInviteCount int `json:"pendingInviteCount"` + + common.NoPKModel +} + +func (ClaudeCodeActivitySummary) TableName() string { + return "_tool_claude_code_activity_summary" +} diff --git a/backend/plugins/claude_code/models/chat_project.go b/backend/plugins/claude_code/models/chat_project.go new file mode 100644 index 00000000000..63eaff010de --- /dev/null +++ b/backend/plugins/claude_code/models/chat_project.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClaudeCodeChatProject captures per-project daily usage from the +// /v1/organizations/analytics/apps/chat/projects endpoint. +type ClaudeCodeChatProject struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + ProjectId string `gorm:"primaryKey;type:varchar(255)" json:"projectId"` + + ProjectName string `json:"projectName" gorm:"type:varchar(255)"` + DistinctUserCount int `json:"distinctUserCount"` + ConversationCount int `json:"conversationCount"` + MessageCount int `json:"messageCount"` + CreatedAt time.Time `json:"createdAt"` + CreatedById string `json:"createdById" gorm:"type:varchar(255)"` + CreatedByEmail string `json:"createdByEmail" gorm:"type:varchar(255);index"` + + common.NoPKModel +} + +func (ClaudeCodeChatProject) TableName() string { + return "_tool_claude_code_chat_project" +} diff --git a/backend/plugins/claude_code/models/connection.go b/backend/plugins/claude_code/models/connection.go new file mode 100644 index 00000000000..700b611ae13 --- /dev/null +++ b/backend/plugins/claude_code/models/connection.go @@ -0,0 +1,203 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const ( + // DefaultEndpoint is the Anthropic API base URL. + DefaultEndpoint = "https://api.anthropic.com" + // DefaultRateLimitPerHour is a conservative default for Admin API calls. + DefaultRateLimitPerHour = 1000 + // AnthropicVersion is the required API version header value. + AnthropicVersion = "2023-06-01" +) + +// CustomHeader represents a single HTTP header key-value pair for middleware authentication. +type CustomHeader struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// ClaudeCodeConn stores Anthropic Claude Code connection settings. +type ClaudeCodeConn struct { + helper.RestConnection `mapstructure:",squash"` + + Token string `mapstructure:"token" json:"token"` + Organization string `mapstructure:"organization" json:"organization" gorm:"type:varchar(255)"` + CustomHeaders []CustomHeader `mapstructure:"customHeaders" json:"customHeaders" gorm:"type:json;serializer:json"` +} + +func (conn *ClaudeCodeConn) HasUsableCustomHeaders() bool { + if conn == nil { + return false + } + for _, header := range conn.CustomHeaders { + if strings.TrimSpace(header.Key) != "" && strings.TrimSpace(header.Value) != "" { + return true + } + } + return false +} + +// IsConsoleApiKey returns true when the token is a Claude Console Admin API key. +// Console keys (sk-ant-admin01-...) have full Admin API access and use +// /v1/organizations/usage_report/claude_code instead of the Enterprise +// /v1/organizations/analytics/* endpoints. +func (conn *ClaudeCodeConn) IsConsoleApiKey() bool { + return strings.HasPrefix(strings.TrimSpace(conn.Token), "sk-ant-admin01-") +} + +func (conn *ClaudeCodeConn) HasIncompleteCustomHeaders() bool { + if conn == nil { + return false + } + for _, header := range conn.CustomHeaders { + hasKey := strings.TrimSpace(header.Key) != "" + hasValue := strings.TrimSpace(header.Value) != "" + if hasKey != hasValue { + return true + } + } + return false +} + +// SetupAuthentication implements plugin.ApiAuthenticator so helper.NewApiClientFromConnection +// can attach the required headers for Anthropic Admin API requests. +// +// When Token is set, the standard Anthropic auth headers (x-api-key, anthropic-version) are +// added. When connecting through a middleware, leave Token empty and supply CustomHeaders instead. +func (conn *ClaudeCodeConn) SetupAuthentication(request *http.Request) errors.Error { + if conn == nil { + return errors.BadInput.New("connection is required") + } + + request.Header.Set("User-Agent", "DevLake/1.0.0") + + if key := strings.TrimSpace(conn.Token); key != "" { + request.Header.Set("x-api-key", key) + request.Header.Set("anthropic-version", AnthropicVersion) + } + + for _, h := range conn.CustomHeaders { + if strings.TrimSpace(h.Key) != "" && strings.TrimSpace(h.Value) != "" { + request.Header.Set(h.Key, h.Value) + } + } + + return nil +} + +// Sanitize returns a copy of the conn with sensitive fields masked. +func (conn *ClaudeCodeConn) Sanitize() ClaudeCodeConn { + if conn == nil { + return ClaudeCodeConn{} + } + clone := *conn + clone.Token = utils.SanitizeString(clone.Token) + sanitizedHeaders := make([]CustomHeader, len(clone.CustomHeaders)) + for i, h := range clone.CustomHeaders { + sanitizedHeaders[i] = CustomHeader{Key: h.Key, Value: utils.SanitizeString(h.Value)} + } + clone.CustomHeaders = sanitizedHeaders + return clone +} + +// ClaudeCodeConnection persists connection details with metadata required by DevLake. +type ClaudeCodeConnection struct { + helper.BaseConnection `mapstructure:",squash"` + ClaudeCodeConn `mapstructure:",squash"` +} + +func (ClaudeCodeConnection) TableName() string { + return "_tool_claude_code_connections" +} + +func (connection ClaudeCodeConnection) Sanitize() ClaudeCodeConnection { + connection.ClaudeCodeConn = connection.ClaudeCodeConn.Sanitize() + return connection +} + +// Normalize applies default connection values where necessary. +func (connection *ClaudeCodeConnection) Normalize() { + if connection == nil { + return + } + if connection.Endpoint == "" { + connection.Endpoint = DefaultEndpoint + } + if connection.RateLimitPerHour <= 0 { + connection.RateLimitPerHour = DefaultRateLimitPerHour + } + normalizedHeaders := make([]CustomHeader, 0, len(connection.CustomHeaders)) + for _, header := range connection.CustomHeaders { + key := strings.TrimSpace(header.Key) + value := strings.TrimSpace(header.Value) + if key == "" && value == "" { + continue + } + normalizedHeaders = append(normalizedHeaders, CustomHeader{Key: key, Value: header.Value}) + } + connection.CustomHeaders = normalizedHeaders +} + +// MergeFromRequest applies a partial update from an HTTP PATCH body, +// preserving original secret values (Token and custom header values) when +// the caller sends back sanitized placeholders. +func (connection *ClaudeCodeConnection) MergeFromRequest(target *ClaudeCodeConnection, body map[string]interface{}) error { + if target == nil { + return nil + } + originalKey := target.Token + originalHeaders := append([]CustomHeader(nil), target.CustomHeaders...) + originalHeaderValues := make(map[string]string, len(originalHeaders)) + for _, h := range originalHeaders { + originalHeaderValues[h.Key] = h.Value + } + + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + + sanitizedOriginal := utils.SanitizeString(originalKey) + if target.Token == "" || target.Token == sanitizedOriginal { + target.Token = originalKey + } + + for i, h := range target.CustomHeaders { + if orig, ok := originalHeaderValues[h.Key]; ok { + if h.Value == "" || h.Value == utils.SanitizeString(orig) { + target.CustomHeaders[i].Value = orig + continue + } + } + + if i < len(originalHeaders) && h.Value != "" && h.Value == utils.SanitizeString(originalHeaders[i].Value) { + target.CustomHeaders[i].Value = originalHeaders[i].Value + } + } + + return nil +} diff --git a/backend/plugins/claude_code/models/connection_test.go b/backend/plugins/claude_code/models/connection_test.go new file mode 100644 index 00000000000..1cb0d55f54b --- /dev/null +++ b/backend/plugins/claude_code/models/connection_test.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apache/incubator-devlake/core/utils" +) + +const ( + testHeaderKey = "Ocp-Apim-Subscription-Key" + testHeaderValue = "secret-key" +) + +func TestMergeFromRequestPreservesSanitizedSecrets(t *testing.T) { + connection := &ClaudeCodeConnection{ + ClaudeCodeConn: ClaudeCodeConn{ + Organization: "anthropic-labs", + Token: "sk-ant-example", + CustomHeaders: []CustomHeader{ + {Key: testHeaderKey, Value: testHeaderValue}, + }, + }, + } + + body := map[string]interface{}{ + "token": utils.SanitizeString(connection.Token), + "customHeaders": []map[string]interface{}{ + { + "key": connection.CustomHeaders[0].Key, + "value": utils.SanitizeString(connection.CustomHeaders[0].Value), + }, + }, + } + + err := (&ClaudeCodeConnection{}).MergeFromRequest(connection, body) + + assert.NoError(t, err) + assert.Equal(t, "sk-ant-example", connection.Token) + assert.Equal(t, []CustomHeader{{Key: testHeaderKey, Value: testHeaderValue}}, connection.CustomHeaders) +} + +func TestMergeFromRequestPreservesSanitizedHeaderValueWhenKeyChanges(t *testing.T) { + connection := &ClaudeCodeConnection{ + ClaudeCodeConn: ClaudeCodeConn{ + Organization: "anthropic-labs", + CustomHeaders: []CustomHeader{ + {Key: testHeaderKey, Value: testHeaderValue}, + }, + }, + } + + body := map[string]interface{}{ + "customHeaders": []map[string]interface{}{ + { + "key": "X-Subscription-Key", + "value": utils.SanitizeString(connection.CustomHeaders[0].Value), + }, + }, + } + + err := (&ClaudeCodeConnection{}).MergeFromRequest(connection, body) + + assert.NoError(t, err) + assert.Equal(t, []CustomHeader{{Key: "X-Subscription-Key", Value: testHeaderValue}}, connection.CustomHeaders) +} diff --git a/backend/plugins/claude_code/models/connector_usage.go b/backend/plugins/claude_code/models/connector_usage.go new file mode 100644 index 00000000000..3dca5305e46 --- /dev/null +++ b/backend/plugins/claude_code/models/connector_usage.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClaudeCodeConnectorUsage captures per-connector daily usage from the +// /v1/organizations/analytics/connectors endpoint. +type ClaudeCodeConnectorUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + ConnectorName string `gorm:"primaryKey;type:varchar(255)" json:"connectorName"` + + DistinctUserCount int `json:"distinctUserCount"` + ChatConversationCount int `json:"chatConversationCount"` + CCSessionCount int `json:"ccSessionCount"` + + common.NoPKModel +} + +func (ClaudeCodeConnectorUsage) TableName() string { + return "_tool_claude_code_connector_usage" +} diff --git a/backend/plugins/claude_code/models/migrationscripts/20260309_initialize.go b/backend/plugins/claude_code/models/migrationscripts/20260309_initialize.go new file mode 100644 index 00000000000..8897bc69a8f --- /dev/null +++ b/backend/plugins/claude_code/models/migrationscripts/20260309_initialize.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addClaudeCodeInitialTables creates the initial Claude Code tool-layer tables. +type addClaudeCodeInitialTables struct{} + +func (script *addClaudeCodeInitialTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &claudeCodeConnection20260309{}, + &claudeCodeScope20260309{}, + &claudeCodeScopeConfig20260309{}, + ) +} + +type claudeCodeConnection20260309 struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + Endpoint string `gorm:"type:varchar(255)" json:"endpoint"` + Proxy string `gorm:"type:varchar(255)" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `json:"token"` + Organization string `gorm:"type:varchar(255)" json:"organization"` +} + +func (claudeCodeConnection20260309) TableName() string { + return "_tool_claude_code_connections" +} + +type claudeCodeScope20260309 struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + Id string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Organization string `json:"organization" gorm:"type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" gorm:"type:varchar(255)"` +} + +func (claudeCodeScope20260309) TableName() string { + return "_tool_claude_code_scopes" +} + +type claudeCodeScopeConfig20260309 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities" mapstructure:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex" validate:"required"` +} + +func (claudeCodeScopeConfig20260309) TableName() string { + return "_tool_claude_code_scope_configs" +} + +func (*addClaudeCodeInitialTables) Version() uint64 { + return 20260309000000 +} + +func (*addClaudeCodeInitialTables) Name() string { + return "claude-code init tables" +} diff --git a/backend/plugins/claude_code/models/migrationscripts/20260319_add_custom_headers.go b/backend/plugins/claude_code/models/migrationscripts/20260319_add_custom_headers.go new file mode 100644 index 00000000000..ba9db8e6dc1 --- /dev/null +++ b/backend/plugins/claude_code/models/migrationscripts/20260319_add_custom_headers.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addClaudeCodeCustomHeaders adds the custom_headers column to the connections table. +type addClaudeCodeCustomHeaders struct{} + +func (script *addClaudeCodeCustomHeaders) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &claudeCodeConnection20260319{}, + ) +} + +type claudeCodeConnection20260319 struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + Endpoint string `gorm:"type:varchar(255)" json:"endpoint"` + Proxy string `gorm:"type:varchar(255)" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `json:"token"` + Organization string `gorm:"type:varchar(255)" json:"organization"` + CustomHeaders string `gorm:"type:json" json:"customHeaders"` +} + +func (claudeCodeConnection20260319) TableName() string { + return "_tool_claude_code_connections" +} + +func (*addClaudeCodeCustomHeaders) Version() uint64 { + return 20260319000000 +} + +func (*addClaudeCodeCustomHeaders) Name() string { + return "claude-code add custom headers to connections" +} diff --git a/backend/plugins/claude_code/models/migrationscripts/20260319_replace_analytics_tables.go b/backend/plugins/claude_code/models/migrationscripts/20260319_replace_analytics_tables.go new file mode 100644 index 00000000000..0c089e64abc --- /dev/null +++ b/backend/plugins/claude_code/models/migrationscripts/20260319_replace_analytics_tables.go @@ -0,0 +1,207 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// replaceClaudeCodeAnalyticsTables creates the five analytics endpoint tables +// and their raw ingestion tables for the updated API surface. +type replaceClaudeCodeAnalyticsTables struct{} + +func (*replaceClaudeCodeAnalyticsTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &ccUserActivity20260319{}, + &ccActivitySummary20260319{}, + &ccChatProject20260319{}, + &ccSkillUsage20260319{}, + &ccConnectorUsage20260319{}, + &ccRawUserActivity20260319{}, + &ccRawActivitySummary20260319{}, + &ccRawChatProject20260319{}, + &ccRawSkillUsage20260319{}, + &ccRawConnectorUsage20260319{}, + ) +} + +func (*replaceClaudeCodeAnalyticsTables) Version() uint64 { return 20260319000001 } +func (*replaceClaudeCodeAnalyticsTables) Name() string { + return "claude-code replace deprecated analytics tables" +} + +// ── Tool-layer table snapshots ──────────────────────────────────────────────── + +type ccUserActivity20260319 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + UserId string `gorm:"primaryKey;type:varchar(255)"` + UserEmail string `gorm:"type:varchar(255);index"` + + ChatConversationCount int + ChatMessageCount int + ChatProjectsCreatedCount int + ChatProjectsUsedCount int + ChatFilesUploadedCount int + ChatArtifactsCreatedCount int + ChatThinkingMessageCount int + ChatSkillsUsedCount int + ChatConnectorsUsedCount int + + CCCommitCount int + CCPullRequestCount int + CCLinesAdded int + CCLinesRemoved int + CCSessionCount int + + EditToolAccepted int + EditToolRejected int + MultiEditToolAccepted int + MultiEditToolRejected int + WriteToolAccepted int + WriteToolRejected int + NotebookEditToolAccepted int + NotebookEditToolRejected int + + WebSearchCount int + archived.NoPKModel +} + +func (ccUserActivity20260319) TableName() string { return "_tool_claude_code_user_activity" } + +type ccActivitySummary20260319 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + DailyActiveUserCount int + WeeklyActiveUserCount int + MonthlyActiveUserCount int + AssignedSeatCount int + PendingInviteCount int + archived.NoPKModel +} + +func (ccActivitySummary20260319) TableName() string { return "_tool_claude_code_activity_summary" } + +type ccChatProject20260319 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + ProjectId string `gorm:"primaryKey;type:varchar(255)"` + ProjectName string `gorm:"type:varchar(255)"` + DistinctUserCount int + ConversationCount int + MessageCount int + CreatedAt time.Time + CreatedById string `gorm:"type:varchar(255)"` + CreatedByEmail string `gorm:"type:varchar(255);index"` + archived.NoPKModel +} + +func (ccChatProject20260319) TableName() string { return "_tool_claude_code_chat_project" } + +type ccSkillUsage20260319 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + SkillName string `gorm:"primaryKey;type:varchar(255)"` + DistinctUserCount int + ChatConversationCount int + CCSessionCount int + archived.NoPKModel +} + +func (ccSkillUsage20260319) TableName() string { return "_tool_claude_code_skill_usage" } + +type ccConnectorUsage20260319 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + ConnectorName string `gorm:"primaryKey;type:varchar(255)"` + DistinctUserCount int + ChatConversationCount int + CCSessionCount int + archived.NoPKModel +} + +func (ccConnectorUsage20260319) TableName() string { return "_tool_claude_code_connector_usage" } + +// ── Raw table snapshots ────────────────────────────────────────────────────── + +type ccRawUserActivity20260319 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (ccRawUserActivity20260319) TableName() string { return "_raw_claude_code_user_activity" } + +type ccRawActivitySummary20260319 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (ccRawActivitySummary20260319) TableName() string { return "_raw_claude_code_activity_summary" } + +type ccRawChatProject20260319 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (ccRawChatProject20260319) TableName() string { return "_raw_claude_code_chat_project" } + +type ccRawSkillUsage20260319 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (ccRawSkillUsage20260319) TableName() string { return "_raw_claude_code_skill_usage" } + +type ccRawConnectorUsage20260319 struct { + ID uint64 `gorm:"primaryKey"` + Params string `gorm:"type:varchar(255);index"` + Data []byte + Url string + Input json.RawMessage `gorm:"type:json"` + CreatedAt time.Time `gorm:"index"` +} + +func (ccRawConnectorUsage20260319) TableName() string { return "_raw_claude_code_connector_usage" } diff --git a/backend/plugins/claude_code/models/migrationscripts/register.go b/backend/plugins/claude_code/models/migrationscripts/register.go new file mode 100644 index 00000000000..e62d15e156b --- /dev/null +++ b/backend/plugins/claude_code/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import "github.com/apache/incubator-devlake/core/plugin" + +// All returns the ordered list of migration scripts for the Claude Code plugin. +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addClaudeCodeInitialTables), + new(addClaudeCodeCustomHeaders), + new(replaceClaudeCodeAnalyticsTables), + } +} diff --git a/backend/plugins/claude_code/models/models.go b/backend/plugins/claude_code/models/models.go new file mode 100644 index 00000000000..c1ee7d8042f --- /dev/null +++ b/backend/plugins/claude_code/models/models.go @@ -0,0 +1,34 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/dal" + +// GetTablesInfo returns the list of tool-layer tables managed by the Claude Code plugin. +func GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &ClaudeCodeConnection{}, + &ClaudeCodeScope{}, + &ClaudeCodeScopeConfig{}, + &ClaudeCodeUserActivity{}, + &ClaudeCodeActivitySummary{}, + &ClaudeCodeChatProject{}, + &ClaudeCodeSkillUsage{}, + &ClaudeCodeConnectorUsage{}, + } +} diff --git a/backend/plugins/claude_code/models/scope.go b/backend/plugins/claude_code/models/scope.go new file mode 100644 index 00000000000..9f085a5fe95 --- /dev/null +++ b/backend/plugins/claude_code/models/scope.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +// ClaudeCodeScope represents an organization-level collection scope. +type ClaudeCodeScope struct { + common.Scope `mapstructure:",squash"` + Id string `json:"id" mapstructure:"id" gorm:"primaryKey;type:varchar(255)"` + Organization string `json:"organization" mapstructure:"organization" gorm:"type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" mapstructure:"fullName" gorm:"type:varchar(255)"` +} + +func (ClaudeCodeScope) TableName() string { + return "_tool_claude_code_scopes" +} + +func (s *ClaudeCodeScope) BeforeSave(tx *gorm.DB) error { + if s == nil { + return nil + } + + s.Id = strings.TrimSpace(s.Id) + s.Organization = strings.TrimSpace(s.Organization) + s.Name = strings.TrimSpace(s.Name) + s.FullName = strings.TrimSpace(s.FullName) + + if s.Organization == "" { + s.Organization = s.Id + } + if s.Id == "" { + s.Id = s.Organization + } + if s.Name == "" { + s.Name = s.ScopeName() + } + if s.FullName == "" { + s.FullName = s.ScopeFullName() + } + + return nil + +} + +func (s ClaudeCodeScope) ScopeId() string { + return s.Id +} + +func (s ClaudeCodeScope) ScopeName() string { + if s.Name != "" { + return s.Name + } + if s.Organization != "" { + return s.Organization + } + return s.Id +} + +func (s ClaudeCodeScope) ScopeFullName() string { + if s.FullName != "" { + return s.FullName + } + return s.ScopeName() +} + +func (s ClaudeCodeScope) ScopeParams() interface{} { + return &ClaudeCodeScopeParams{ + ConnectionId: s.ConnectionId, + ScopeId: s.Id, + } +} + +// ClaudeCodeScopeParams is returned for blueprint configuration. +type ClaudeCodeScopeParams struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` +} + +var _ plugin.ToolLayerScope = (*ClaudeCodeScope)(nil) diff --git a/backend/plugins/claude_code/models/scope_config.go b/backend/plugins/claude_code/models/scope_config.go new file mode 100644 index 00000000000..22f54f3a983 --- /dev/null +++ b/backend/plugins/claude_code/models/scope_config.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScopeConfig = (*ClaudeCodeScopeConfig)(nil) + +// ClaudeCodeScopeConfig contains configuration for Claude Code data scope. +type ClaudeCodeScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (ClaudeCodeScopeConfig) TableName() string { + return "_tool_claude_code_scope_configs" +} + +// GetConnectionId implements plugin.ToolLayerScopeConfig. +func (sc ClaudeCodeScopeConfig) GetConnectionId() uint64 { + return sc.ConnectionId +} diff --git a/backend/plugins/claude_code/models/skill_usage.go b/backend/plugins/claude_code/models/skill_usage.go new file mode 100644 index 00000000000..3ca4d6fdc8e --- /dev/null +++ b/backend/plugins/claude_code/models/skill_usage.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClaudeCodeSkillUsage captures per-skill daily usage from the +// /v1/organizations/analytics/skills endpoint. +type ClaudeCodeSkillUsage struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + SkillName string `gorm:"primaryKey;type:varchar(255)" json:"skillName"` + + DistinctUserCount int `json:"distinctUserCount"` + ChatConversationCount int `json:"chatConversationCount"` + CCSessionCount int `json:"ccSessionCount"` + + common.NoPKModel +} + +func (ClaudeCodeSkillUsage) TableName() string { + return "_tool_claude_code_skill_usage" +} diff --git a/backend/plugins/claude_code/models/user_activity.go b/backend/plugins/claude_code/models/user_activity.go new file mode 100644 index 00000000000..6385ae82951 --- /dev/null +++ b/backend/plugins/claude_code/models/user_activity.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// ClaudeCodeUserActivity captures per-user daily engagement metrics from the +// /v1/organizations/analytics/users endpoint. +type ClaudeCodeUserActivity struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + + UserEmail string `json:"userEmail" gorm:"type:varchar(255);index"` + + // Claude.ai (chat) metrics + ChatConversationCount int `json:"chatConversationCount"` + ChatMessageCount int `json:"chatMessageCount"` + ChatProjectsCreatedCount int `json:"chatProjectsCreatedCount"` + ChatProjectsUsedCount int `json:"chatProjectsUsedCount"` + ChatFilesUploadedCount int `json:"chatFilesUploadedCount"` + ChatArtifactsCreatedCount int `json:"chatArtifactsCreatedCount"` + ChatThinkingMessageCount int `json:"chatThinkingMessageCount"` + ChatSkillsUsedCount int `json:"chatSkillsUsedCount"` + ChatConnectorsUsedCount int `json:"chatConnectorsUsedCount"` + + // Claude Code core metrics + CCCommitCount int `json:"ccCommitCount"` + CCPullRequestCount int `json:"ccPullRequestCount"` + CCLinesAdded int `json:"ccLinesAdded"` + CCLinesRemoved int `json:"ccLinesRemoved"` + CCSessionCount int `json:"ccSessionCount"` + + // Claude Code tool actions + EditToolAccepted int `json:"editToolAccepted"` + EditToolRejected int `json:"editToolRejected"` + MultiEditToolAccepted int `json:"multiEditToolAccepted"` + MultiEditToolRejected int `json:"multiEditToolRejected"` + WriteToolAccepted int `json:"writeToolAccepted"` + WriteToolRejected int `json:"writeToolRejected"` + NotebookEditToolAccepted int `json:"notebookEditToolAccepted"` + NotebookEditToolRejected int `json:"notebookEditToolRejected"` + + WebSearchCount int `json:"webSearchCount"` + + common.NoPKModel +} + +func (ClaudeCodeUserActivity) TableName() string { + return "_tool_claude_code_user_activity" +} diff --git a/backend/plugins/claude_code/service/connection_test_helper.go b/backend/plugins/claude_code/service/connection_test_helper.go new file mode 100644 index 00000000000..97581ce379e --- /dev/null +++ b/backend/plugins/claude_code/service/connection_test_helper.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + stdctx "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// TestConnectionResult represents the payload returned by the connection test endpoints. +type TestConnectionResult struct { + Success bool `json:"success"` + Message string `json:"message"` + OrganizationId string `json:"organizationId,omitempty"` +} + +// TestConnection exercises the Claude Code Analytics API to validate credentials. +// It makes a minimal request with limit=1 to confirm the connection is valid. +func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models.ClaudeCodeConnection) (*TestConnectionResult, errors.Error) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + connection.Normalize() + + hasToken := strings.TrimSpace(connection.Token) != "" + if connection.HasIncompleteCustomHeaders() { + return nil, errors.BadInput.New("custom headers must include both key and value") + } + hasCustomHeaders := connection.HasUsableCustomHeaders() + if !hasToken && !hasCustomHeaders { + return nil, errors.BadInput.New("either token or at least one custom header is required") + } + if strings.TrimSpace(connection.Organization) == "" { + return nil, errors.BadInput.New("organization is required") + } + + apiClient, err := helper.NewApiClientFromConnection(ctx, br, connection) + if err != nil { + return nil, err + } + + // Use a date safely outside the analytics availability lag window for the test request. + testDate := time.Now().UTC().AddDate(0, 0, -7).Format("2006-01-02") + endpoint := fmt.Sprintf("v1/organizations/usage_report/claude_code?starting_at=%s&limit=1", testDate) + + res, err := apiClient.Get(endpoint, nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to reach Claude Code Analytics API") + } + defer res.Body.Close() + + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return &TestConnectionResult{ + Success: false, + Message: fmt.Sprintf("authentication failed (HTTP %d): verify your API credentials", res.StatusCode), + }, nil + } + + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return &TestConnectionResult{ + Success: false, + Message: fmt.Sprintf("unexpected status %d: %s", res.StatusCode, string(body)), + }, nil + } + + // Parse the response to confirm it's a valid analytics payload. + body, readErr := io.ReadAll(res.Body) + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read response body") + } + + var response struct { + Data json.RawMessage `json:"data"` + HasMore bool `json:"has_more"` + } + if jsonErr := json.Unmarshal(body, &response); jsonErr != nil { + return &TestConnectionResult{ + Success: false, + Message: fmt.Sprintf("failed to parse response: %v", jsonErr), + }, nil + } + + return &TestConnectionResult{ + Success: true, + Message: "Connection validated successfully", + OrganizationId: connection.Organization, + }, nil +} diff --git a/backend/plugins/claude_code/tasks/activity_summary_collector.go b/backend/plugins/claude_code/tasks/activity_summary_collector.go new file mode 100644 index 00000000000..79728f19090 --- /dev/null +++ b/backend/plugins/claude_code/tasks/activity_summary_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectActivitySummary collects daily organisation-level summaries from +// /v1/organizations/analytics/summaries in 31-day chunks. +func CollectActivitySummary(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping activity summary collection") + return nil + } + + if connection.IsConsoleApiKey() { + taskCtx.GetLogger().Info("Console API key detected, skipping activity summary collection (no equivalent endpoint)") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawActivitySummaryTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/summaries", + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + start, until := computeUsageDateRange(time.Now().UTC(), collector.GetSince()) + rangeIter := newClaudeCodeDateRangeIterator(start, until, claudeCodeSummaryMaxDays) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: rangeIter, + PageSize: 1, + Incremental: true, + UrlTemplate: "v1/organizations/analytics/summaries", + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*claudeCodeDateRangeInput) + query := url.Values{} + query.Set("starting_date", input.StartDate) + query.Set("ending_date", input.EndDate) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + page, err := parseClaudeCodeUsagePage(res) + if err != nil { + return nil, err + } + return page.Data, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/claude_code/tasks/activity_summary_extractor.go b/backend/plugins/claude_code/tasks/activity_summary_extractor.go new file mode 100644 index 00000000000..8e63e8d8c2d --- /dev/null +++ b/backend/plugins/claude_code/tasks/activity_summary_extractor.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// activitySummaryRecord is the JSON shape returned by /v1/organizations/analytics/summaries. +type activitySummaryRecord struct { + StartingDate string `json:"starting_date"` + EndingDate string `json:"ending_date"` + DailyActiveUserCount int `json:"daily_active_user_count"` + WeeklyActiveUserCount int `json:"weekly_active_user_count"` + MonthlyActiveUserCount int `json:"monthly_active_user_count"` + AssignedSeatCount int `json:"assigned_seat_count"` + PendingInviteCount int `json:"pending_invite_count"` +} + +// ExtractActivitySummary parses raw activity summary records into tool-layer tables. +func ExtractActivitySummary(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping activity summary extraction") + return nil + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawActivitySummaryTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/summaries", + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record activitySummaryRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + dateStr := strings.TrimSpace(record.StartingDate) + if dateStr == "" { + return nil, nil + } + t, parseErr := time.Parse("2006-01-02", dateStr) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid starting_date in activity summary") + } + + summary := &models.ClaudeCodeActivitySummary{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: utcDate(t), + DailyActiveUserCount: record.DailyActiveUserCount, + WeeklyActiveUserCount: record.WeeklyActiveUserCount, + MonthlyActiveUserCount: record.MonthlyActiveUserCount, + AssignedSeatCount: record.AssignedSeatCount, + PendingInviteCount: record.PendingInviteCount, + } + return []interface{}{summary}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/claude_code/tasks/api_client.go b/backend/plugins/claude_code/tasks/api_client.go new file mode 100644 index 00000000000..8f3f29da4fb --- /dev/null +++ b/backend/plugins/claude_code/tasks/api_client.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +type nowFunc func() time.Time +type sleepFunc func(time.Duration) + +func handleAnthropicRetryAfter(res *http.Response, logger log.Logger, now nowFunc, sleep sleepFunc) errors.Error { + if res == nil { + return nil + } + if res.StatusCode != http.StatusTooManyRequests { + return nil + } + + if now == nil { + now = time.Now + } + if sleep == nil { + sleep = time.Sleep + } + + wait := parseRetryAfter(res.Header.Get("Retry-After"), now().UTC()) + if wait > 0 { + if logger != nil { + logger.Warn(nil, "Anthropic returned 429; sleeping %s per Retry-After", wait.String()) + } + sleep(wait) + } + return errors.HttpStatus(http.StatusTooManyRequests).New("Anthropic rate limited the request") +} + +// CreateApiClient creates an async API client for Claude Code collectors. +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.ClaudeCodeConnection) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + apiClient.SetHeaders(map[string]string{ + "Accept": "application/json", + }) + + rateLimiter := &helper.ApiRateLimitCalculator{UserRateLimitPerHour: connection.RateLimitPerHour} + asyncClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, rateLimiter) + if err != nil { + return nil, err + } + + apiClient.SetAfterFunction(func(res *http.Response) errors.Error { + return handleAnthropicRetryAfter(res, taskCtx.GetLogger(), time.Now, time.Sleep) + }) + + return asyncClient, nil +} diff --git a/backend/plugins/claude_code/tasks/chat_project_collector.go b/backend/plugins/claude_code/tasks/chat_project_collector.go new file mode 100644 index 00000000000..18062e186b3 --- /dev/null +++ b/backend/plugins/claude_code/tasks/chat_project_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectChatProjects collects per-project daily usage from +// /v1/organizations/analytics/apps/chat/projects. +func CollectChatProjects(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping chat project collection") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawChatProjectTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/apps/chat/projects", + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + start, until := computeUsageDateRange(time.Now().UTC(), collector.GetSince()) + dayIter := newClaudeCodeDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + PageSize: 1, + Incremental: true, + UrlTemplate: "v1/organizations/analytics/apps/chat/projects", + GetNextPageCustomData: getNextClaudeCodePageCursor, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*claudeCodeDayInput) + query := url.Values{} + query.Set("date", input.Day) + query.Set("limit", fmt.Sprintf("%d", claudeCodeApiPageLimit)) + if cursor, ok := reqData.CustomData.(string); ok && strings.TrimSpace(cursor) != "" { + query.Set("page", cursor) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + page, err := parseClaudeCodeUsagePage(res) + if err != nil { + return nil, err + } + return page.Data, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/claude_code/tasks/chat_project_extractor.go b/backend/plugins/claude_code/tasks/chat_project_extractor.go new file mode 100644 index 00000000000..55a27a53c38 --- /dev/null +++ b/backend/plugins/claude_code/tasks/chat_project_extractor.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// chatProjectRecord is the JSON shape returned by /v1/organizations/analytics/apps/chat/projects. +type chatProjectRecord struct { + ProjectName string `json:"project_name"` + ProjectId string `json:"project_id"` + DistinctUserCount int `json:"distinct_user_count"` + DistinctConversationCount int `json:"distinct_conversation_count"` + MessageCount int `json:"message_count"` + CreatedAt string `json:"created_at"` + CreatedBy chatProjectCreator `json:"created_by"` +} + +type chatProjectCreator struct { + Id string `json:"id"` + EmailAddress string `json:"email_address"` +} + +// ExtractChatProjects parses raw chat project records into tool-layer tables. +func ExtractChatProjects(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping chat project extraction") + return nil + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawChatProjectTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/apps/chat/projects", + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record chatProjectRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + date, parseErr := parseAnalyticsDate(row.Input) + if parseErr != nil { + return nil, parseErr + } + + projectId := strings.TrimSpace(record.ProjectId) + if projectId == "" { + return nil, nil + } + + var createdAt time.Time + if ts := strings.TrimSpace(record.CreatedAt); ts != "" { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + createdAt = t.UTC() + } + } + + project := &models.ClaudeCodeChatProject{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + ProjectId: projectId, + ProjectName: strings.TrimSpace(record.ProjectName), + DistinctUserCount: record.DistinctUserCount, + ConversationCount: record.DistinctConversationCount, + MessageCount: record.MessageCount, + CreatedAt: createdAt, + CreatedById: strings.TrimSpace(record.CreatedBy.Id), + CreatedByEmail: strings.TrimSpace(record.CreatedBy.EmailAddress), + } + return []interface{}{project}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/claude_code/tasks/collector_utils.go b/backend/plugins/claude_code/tasks/collector_utils.go new file mode 100644 index 00000000000..1be3a4165b3 --- /dev/null +++ b/backend/plugins/claude_code/tasks/collector_utils.go @@ -0,0 +1,195 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const ( + rawUserActivityTable = "claude_code_user_activity" + rawActivitySummaryTable = "claude_code_activity_summary" + rawChatProjectTable = "claude_code_chat_project" + rawSkillUsageTable = "claude_code_skill_usage" + rawConnectorUsageTable = "claude_code_connector_usage" + + claudeCodeApiPageLimit = 1000 + claudeCodeInitialBackfillDays = 90 + claudeCodeSummaryMaxDays = 31 + claudeCodeAvailabilityLagDays = 3 + claudeCodeDateLayout = "2006-01-02" +) + +// claudeCodeRawParams identifies a set of raw data records for a given connection/scope. +type claudeCodeRawParams struct { + ConnectionId uint64 + ScopeId string + Organization string + Endpoint string +} + +func (p claudeCodeRawParams) GetParams() any { + return p +} + +// claudeCodeUsagePage is the common paginated response envelope used by all analytics endpoints. +type claudeCodeUsagePage struct { + Data []json.RawMessage `json:"data"` + HasMore bool `json:"has_more"` + NextPage *string `json:"next_page"` +} + +// claudeCodeDayInput is the input item for day-based collectors. +type claudeCodeDayInput struct { + Day string `json:"day"` +} + +// claudeCodeDateRangeInput is the input item for the summaries (range-based) collector. +type claudeCodeDateRangeInput struct { + StartDate string `json:"start_date"` + EndDate string `json:"end_date"` +} + +// utcDate truncates a time to midnight UTC. +func utcDate(t time.Time) time.Time { + y, m, d := t.UTC().Date() + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +// computeUsageDateRange returns the [start, until] date range for incremental collection. +// The Anthropic Analytics API only serves data from 2026-01-01 onwards and requires +// dates to be at least three days old, so the current day and previous two days are skipped. +func computeUsageDateRange(now time.Time, since *time.Time) (start, until time.Time) { + apiFloor := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + until = utcDate(now).AddDate(0, 0, -claudeCodeAvailabilityLagDays) + start = until.AddDate(0, 0, -(claudeCodeInitialBackfillDays - 1)) + if since != nil { + start = utcDate(*since).AddDate(0, 0, 1) + minStart := until.AddDate(0, 0, -(claudeCodeInitialBackfillDays - 1)) + if start.Before(minStart) { + start = minStart + } + } + if start.Before(apiFloor) { + start = apiFloor + } + return start, until +} + +// claudeCodeDayIterator iterates over individual calendar days. +type claudeCodeDayIterator struct { + days []claudeCodeDayInput + idx int +} + +func newClaudeCodeDayIterator(start, end time.Time) *claudeCodeDayIterator { + days := make([]claudeCodeDayInput, 0) + for d := start; !d.After(end); d = d.AddDate(0, 0, 1) { + days = append(days, claudeCodeDayInput{Day: d.Format(claudeCodeDateLayout)}) + } + return &claudeCodeDayIterator{days: days} +} + +func (it *claudeCodeDayIterator) HasNext() bool { return it.idx < len(it.days) } +func (it *claudeCodeDayIterator) Close() errors.Error { return nil } +func (it *claudeCodeDayIterator) Fetch() (interface{}, errors.Error) { + if it.idx >= len(it.days) { + return nil, nil + } + day := it.days[it.idx] + it.idx++ + return &day, nil +} + +// claudeCodeDateRangeIterator iterates over date ranges in chunks of up to maxDays days. +type claudeCodeDateRangeIterator struct { + chunks []claudeCodeDateRangeInput + idx int +} + +func newClaudeCodeDateRangeIterator(start, end time.Time, maxDays int) *claudeCodeDateRangeIterator { + chunks := make([]claudeCodeDateRangeInput, 0) + for chunkStart := start; !chunkStart.After(end); { + chunkEnd := chunkStart.AddDate(0, 0, maxDays-1) + if chunkEnd.After(end) { + chunkEnd = end + } + chunks = append(chunks, claudeCodeDateRangeInput{ + StartDate: chunkStart.Format(claudeCodeDateLayout), + EndDate: chunkEnd.AddDate(0, 0, 1).Format(claudeCodeDateLayout), // exclusive end + }) + chunkStart = chunkEnd.AddDate(0, 0, 1) + } + return &claudeCodeDateRangeIterator{chunks: chunks} +} + +func (it *claudeCodeDateRangeIterator) HasNext() bool { return it.idx < len(it.chunks) } +func (it *claudeCodeDateRangeIterator) Close() errors.Error { return nil } +func (it *claudeCodeDateRangeIterator) Fetch() (interface{}, errors.Error) { + if it.idx >= len(it.chunks) { + return nil, nil + } + chunk := it.chunks[it.idx] + it.idx++ + return &chunk, nil +} + +// parseClaudeCodeUsagePage reads and parses a paginated analytics API response. +func parseClaudeCodeUsagePage(res *http.Response) (*claudeCodeUsagePage, errors.Error) { + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to read analytics response") + } + res.Body.Close() + + if res.StatusCode >= http.StatusBadRequest { + snippet := string(body) + if len(snippet) > 500 { + snippet = snippet[:500] + } + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("analytics request failed: %s", snippet)) + } + + var page claudeCodeUsagePage + if err := errors.Convert(json.Unmarshal(body, &page)); err != nil { + return nil, errors.Default.Wrap(err, "failed to decode analytics response") + } + + res.Body = io.NopCloser(strings.NewReader(string(body))) + return &page, nil +} + +// getNextClaudeCodePageCursor returns the next page cursor or ErrFinishCollect. +func getNextClaudeCodePageCursor(_ *helper.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + page, err := parseClaudeCodeUsagePage(prevPageResponse) + if err != nil { + return nil, err + } + if !page.HasMore || page.NextPage == nil || strings.TrimSpace(*page.NextPage) == "" { + return nil, helper.ErrFinishCollect + } + return strings.TrimSpace(*page.NextPage), nil +} diff --git a/backend/plugins/claude_code/tasks/collector_utils_test.go b/backend/plugins/claude_code/tasks/collector_utils_test.go new file mode 100644 index 00000000000..86ba3165679 --- /dev/null +++ b/backend/plugins/claude_code/tasks/collector_utils_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestComputeUsageDateRangeSkipsRecentDays(t *testing.T) { + now := time.Date(2026, 6, 19, 14, 30, 0, 0, time.UTC) + + start, until := computeUsageDateRange(now, nil) + + assert.Equal(t, time.Date(2026, 3, 19, 0, 0, 0, 0, time.UTC), start) + assert.Equal(t, time.Date(2026, 6, 16, 0, 0, 0, 0, time.UTC), until) +} + +func TestComputeUsageDateRangeHonorsApiFloor(t *testing.T) { + now := time.Date(2026, 3, 19, 9, 0, 0, 0, time.UTC) + + start, until := computeUsageDateRange(now, nil) + + assert.Equal(t, time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), start) + assert.Equal(t, time.Date(2026, 3, 16, 0, 0, 0, 0, time.UTC), until) +} + +func TestComputeUsageDateRangeProducesEmptyWindowInsideLag(t *testing.T) { + now := time.Date(2026, 6, 19, 14, 30, 0, 0, time.UTC) + since := time.Date(2026, 6, 16, 12, 0, 0, 0, time.UTC) + + start, until := computeUsageDateRange(now, &since) + + assert.Equal(t, time.Date(2026, 6, 17, 0, 0, 0, 0, time.UTC), start) + assert.Equal(t, time.Date(2026, 6, 16, 0, 0, 0, 0, time.UTC), until) + assert.False(t, newClaudeCodeDayIterator(start, until).HasNext()) + assert.False(t, newClaudeCodeDateRangeIterator(start, until, claudeCodeSummaryMaxDays).HasNext()) +} diff --git a/backend/plugins/claude_code/tasks/connector_usage_collector.go b/backend/plugins/claude_code/tasks/connector_usage_collector.go new file mode 100644 index 00000000000..2b885b1ef57 --- /dev/null +++ b/backend/plugins/claude_code/tasks/connector_usage_collector.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectConnectorUsage collects per-connector daily usage from /v1/organizations/analytics/connectors. +func CollectConnectorUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping connector usage collection") + return nil + } + + if connection.IsConsoleApiKey() { + taskCtx.GetLogger().Info("Console API key detected, skipping connector usage collection (no equivalent endpoint)") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawConnectorUsageTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/connectors", + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + start, until := computeUsageDateRange(time.Now().UTC(), collector.GetSince()) + dayIter := newClaudeCodeDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + PageSize: 1, + Incremental: true, + UrlTemplate: "v1/organizations/analytics/connectors", + GetNextPageCustomData: getNextClaudeCodePageCursor, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*claudeCodeDayInput) + query := url.Values{} + query.Set("date", input.Day) + query.Set("limit", fmt.Sprintf("%d", claudeCodeApiPageLimit)) + if cursor, ok := reqData.CustomData.(string); ok && strings.TrimSpace(cursor) != "" { + query.Set("page", cursor) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + page, err := parseClaudeCodeUsagePage(res) + if err != nil { + return nil, err + } + return page.Data, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/claude_code/tasks/connector_usage_extractor.go b/backend/plugins/claude_code/tasks/connector_usage_extractor.go new file mode 100644 index 00000000000..833ca4619d0 --- /dev/null +++ b/backend/plugins/claude_code/tasks/connector_usage_extractor.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// connectorUsageRecord is the JSON shape returned by /v1/organizations/analytics/connectors. +type connectorUsageRecord struct { + ConnectorName string `json:"connector_name"` + DistinctUserCount int `json:"distinct_user_count"` + ChatMetrics connectorUsageChat `json:"chat_metrics"` + ClaudeCodeMetrics connectorUsageCC `json:"claude_code_metrics"` +} + +type connectorUsageChat struct { + DistinctConversationConnectorUsedCount int `json:"distinct_conversation_connector_used_count"` +} + +type connectorUsageCC struct { + DistinctSessionConnectorUsedCount int `json:"distinct_session_connector_used_count"` +} + +// ExtractConnectorUsage parses raw connector usage records into tool-layer tables. +func ExtractConnectorUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping connector usage extraction") + return nil + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawConnectorUsageTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/connectors", + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record connectorUsageRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + date, parseErr := parseAnalyticsDate(row.Input) + if parseErr != nil { + return nil, parseErr + } + + connectorName := strings.TrimSpace(record.ConnectorName) + if connectorName == "" { + return nil, nil + } + + connector := &models.ClaudeCodeConnectorUsage{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + ConnectorName: connectorName, + DistinctUserCount: record.DistinctUserCount, + ChatConversationCount: record.ChatMetrics.DistinctConversationConnectorUsedCount, + CCSessionCount: record.ClaudeCodeMetrics.DistinctSessionConnectorUsedCount, + } + return []interface{}{connector}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/claude_code/tasks/options.go b/backend/plugins/claude_code/tasks/options.go new file mode 100644 index 00000000000..f9f08f2e2db --- /dev/null +++ b/backend/plugins/claude_code/tasks/options.go @@ -0,0 +1,24 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +// ClaudeCodeOptions defines task-level options passed from pipeline plans. +type ClaudeCodeOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeId string `mapstructure:"scopeId" json:"scopeId"` +} diff --git a/backend/plugins/claude_code/tasks/register.go b/backend/plugins/claude_code/tasks/register.go new file mode 100644 index 00000000000..375076988a3 --- /dev/null +++ b/backend/plugins/claude_code/tasks/register.go @@ -0,0 +1,36 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +// GetSubTaskMetas returns the ordered list of Claude Code subtasks. +func GetSubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + CollectUserActivityMeta, + ExtractUserActivityMeta, + CollectActivitySummaryMeta, + ExtractActivitySummaryMeta, + CollectChatProjectsMeta, + ExtractChatProjectsMeta, + CollectSkillUsageMeta, + ExtractSkillUsageMeta, + CollectConnectorUsageMeta, + ExtractConnectorUsageMeta, + } +} diff --git a/backend/plugins/claude_code/tasks/retry_after.go b/backend/plugins/claude_code/tasks/retry_after.go new file mode 100644 index 00000000000..6b0a36b805e --- /dev/null +++ b/backend/plugins/claude_code/tasks/retry_after.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "strconv" + "time" +) + +func parseRetryAfter(value string, now time.Time) time.Duration { + if value == "" { + return 0 + } + if seconds, err := strconv.Atoi(value); err == nil { + if seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second + } + if t, err := http.ParseTime(value); err == nil { + wait := t.Sub(now) + if wait < 0 { + return 0 + } + return wait + } + return 0 +} diff --git a/backend/plugins/claude_code/tasks/skill_usage_collector.go b/backend/plugins/claude_code/tasks/skill_usage_collector.go new file mode 100644 index 00000000000..dfb87612da4 --- /dev/null +++ b/backend/plugins/claude_code/tasks/skill_usage_collector.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectSkillUsage collects per-skill daily usage from /v1/organizations/analytics/skills. +func CollectSkillUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping skill usage collection") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawSkillUsageTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/skills", + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + start, until := computeUsageDateRange(time.Now().UTC(), collector.GetSince()) + dayIter := newClaudeCodeDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + PageSize: 1, + Incremental: true, + UrlTemplate: "v1/organizations/analytics/skills", + GetNextPageCustomData: getNextClaudeCodePageCursor, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*claudeCodeDayInput) + query := url.Values{} + query.Set("date", input.Day) + query.Set("limit", fmt.Sprintf("%d", claudeCodeApiPageLimit)) + if cursor, ok := reqData.CustomData.(string); ok && strings.TrimSpace(cursor) != "" { + query.Set("page", cursor) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + page, err := parseClaudeCodeUsagePage(res) + if err != nil { + return nil, err + } + return page.Data, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/claude_code/tasks/skill_usage_extractor.go b/backend/plugins/claude_code/tasks/skill_usage_extractor.go new file mode 100644 index 00000000000..72743cee395 --- /dev/null +++ b/backend/plugins/claude_code/tasks/skill_usage_extractor.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// skillUsageRecord is the JSON shape returned by /v1/organizations/analytics/skills. +type skillUsageRecord struct { + SkillName string `json:"skill_name"` + DistinctUserCount int `json:"distinct_user_count"` + ChatMetrics skillUsageChat `json:"chat_metrics"` + ClaudeCodeMetrics skillUsageCC `json:"claude_code_metrics"` +} + +type skillUsageChat struct { + DistinctConversationSkillUsedCount int `json:"distinct_conversation_skill_used_count"` +} + +type skillUsageCC struct { + DistinctSessionSkillUsedCount int `json:"distinct_session_skill_used_count"` +} + +// ExtractSkillUsage parses raw skill usage records into tool-layer tables. +func ExtractSkillUsage(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping skill usage extraction") + return nil + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawSkillUsageTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: "analytics/skills", + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var record skillUsageRecord + if err := errors.Convert(json.Unmarshal(row.Data, &record)); err != nil { + return nil, err + } + + date, parseErr := parseAnalyticsDate(row.Input) + if parseErr != nil { + return nil, parseErr + } + + skillName := strings.TrimSpace(record.SkillName) + if skillName == "" { + return nil, nil + } + + skill := &models.ClaudeCodeSkillUsage{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + SkillName: skillName, + DistinctUserCount: record.DistinctUserCount, + ChatConversationCount: record.ChatMetrics.DistinctConversationSkillUsedCount, + CCSessionCount: record.ClaudeCodeMetrics.DistinctSessionSkillUsedCount, + } + return []interface{}{skill}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/claude_code/tasks/subtasks.go b/backend/plugins/claude_code/tasks/subtasks.go new file mode 100644 index 00000000000..82fd4ae63e0 --- /dev/null +++ b/backend/plugins/claude_code/tasks/subtasks.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +var CollectUserActivityMeta = plugin.SubTaskMeta{ + Name: "collectUserActivity", + EntryPoint: CollectUserActivity, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-user daily engagement metrics from the Anthropic Analytics API", +} + +var ExtractUserActivityMeta = plugin.SubTaskMeta{ + Name: "extractUserActivity", + EntryPoint: ExtractUserActivity, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-user daily engagement metrics into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectUserActivityMeta}, +} + +var CollectActivitySummaryMeta = plugin.SubTaskMeta{ + Name: "collectActivitySummary", + EntryPoint: CollectActivitySummary, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect organisation-level daily activity summaries from the Anthropic Analytics API", +} + +var ExtractActivitySummaryMeta = plugin.SubTaskMeta{ + Name: "extractActivitySummary", + EntryPoint: ExtractActivitySummary, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract organisation-level daily activity summaries into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectActivitySummaryMeta}, +} + +var CollectChatProjectsMeta = plugin.SubTaskMeta{ + Name: "collectChatProjects", + EntryPoint: CollectChatProjects, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-project daily chat usage from the Anthropic Analytics API", +} + +var ExtractChatProjectsMeta = plugin.SubTaskMeta{ + Name: "extractChatProjects", + EntryPoint: ExtractChatProjects, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-project daily chat usage into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectChatProjectsMeta}, +} + +var CollectSkillUsageMeta = plugin.SubTaskMeta{ + Name: "collectSkillUsage", + EntryPoint: CollectSkillUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-skill daily usage from the Anthropic Analytics API", +} + +var ExtractSkillUsageMeta = plugin.SubTaskMeta{ + Name: "extractSkillUsage", + EntryPoint: ExtractSkillUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-skill daily usage into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectSkillUsageMeta}, +} + +var CollectConnectorUsageMeta = plugin.SubTaskMeta{ + Name: "collectConnectorUsage", + EntryPoint: CollectConnectorUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect per-connector daily usage from the Anthropic Analytics API", +} + +var ExtractConnectorUsageMeta = plugin.SubTaskMeta{ + Name: "extractConnectorUsage", + EntryPoint: ExtractConnectorUsage, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract per-connector daily usage into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectConnectorUsageMeta}, +} diff --git a/backend/plugins/claude_code/tasks/task_data.go b/backend/plugins/claude_code/tasks/task_data.go new file mode 100644 index 00000000000..7c72e5261b9 --- /dev/null +++ b/backend/plugins/claude_code/tasks/task_data.go @@ -0,0 +1,26 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/plugins/claude_code/models" + +// ClaudeCodeTaskData stores runtime dependencies for subtasks. +type ClaudeCodeTaskData struct { + Options *ClaudeCodeOptions + Connection *models.ClaudeCodeConnection +} diff --git a/backend/plugins/claude_code/tasks/user_activity_collector.go b/backend/plugins/claude_code/tasks/user_activity_collector.go new file mode 100644 index 00000000000..7239390be52 --- /dev/null +++ b/backend/plugins/claude_code/tasks/user_activity_collector.go @@ -0,0 +1,116 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// CollectUserActivity collects per-user daily engagement metrics. +// For Claude Enterprise organizations it calls /v1/organizations/analytics/users. +// For Claude Console organizations (sk-ant-admin01-... keys) it calls +// /v1/organizations/usage_report/claude_code. +func CollectUserActivity(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping user activity collection") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + isConsole := connection.IsConsoleApiKey() + endpoint := "analytics/users" + urlTemplate := "v1/organizations/analytics/users" + if isConsole { + endpoint = "usage_report/claude_code" + urlTemplate = "v1/organizations/usage_report/claude_code" + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserActivityTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + start, until := computeUsageDateRange(time.Now().UTC(), collector.GetSince()) + dayIter := newClaudeCodeDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + PageSize: 1, + Incremental: true, + UrlTemplate: urlTemplate, + GetNextPageCustomData: getNextClaudeCodePageCursor, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*claudeCodeDayInput) + query := url.Values{} + if isConsole { + query.Set("starting_at", input.Day) + } else { + query.Set("date", input.Day) + } + query.Set("limit", fmt.Sprintf("%d", claudeCodeApiPageLimit)) + if cursor, ok := reqData.CustomData.(string); ok && strings.TrimSpace(cursor) != "" { + query.Set("page", cursor) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + page, err := parseClaudeCodeUsagePage(res) + if err != nil { + return nil, err + } + return page.Data, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/claude_code/tasks/user_activity_extractor.go b/backend/plugins/claude_code/tasks/user_activity_extractor.go new file mode 100644 index 00000000000..4050a7c2b1f --- /dev/null +++ b/backend/plugins/claude_code/tasks/user_activity_extractor.go @@ -0,0 +1,285 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/claude_code/models" +) + +// ── Enterprise API response types (/v1/organizations/analytics/users) ────────── + +// userActivityRecord is the JSON shape returned by /v1/organizations/analytics/users. +type userActivityRecord struct { + User userActivityUser `json:"user"` + ChatMetrics userActivityChat `json:"chat_metrics"` + ClaudeCodeMetrics userActivityCC `json:"claude_code_metrics"` + WebSearchCount int `json:"web_search_count"` +} + +type userActivityUser struct { + Id string `json:"id"` + EmailAddress string `json:"email_address"` +} + +type userActivityChat struct { + DistinctConversationCount int `json:"distinct_conversation_count"` + MessageCount int `json:"message_count"` + DistinctProjectsCreatedCount int `json:"distinct_projects_created_count"` + DistinctProjectsUsedCount int `json:"distinct_projects_used_count"` + DistinctFilesUploadedCount int `json:"distinct_files_uploaded_count"` + DistinctArtifactsCreatedCount int `json:"distinct_artifacts_created_count"` + ThinkingMessageCount int `json:"thinking_message_count"` + DistinctSkillsUsedCount int `json:"distinct_skills_used_count"` + ConnectorsUsedCount int `json:"connectors_used_count"` +} + +type userActivityCC struct { + CoreMetrics userActivityCCCore `json:"core_metrics"` + ToolActions userActivityCCTools `json:"tool_actions"` +} + +type userActivityCCCore struct { + CommitCount int `json:"commit_count"` + PullRequestCount int `json:"pull_request_count"` + LinesOfCode userActivityLines `json:"lines_of_code"` + DistinctSessionCount int `json:"distinct_session_count"` +} + +type userActivityLines struct { + AddedCount int `json:"added_count"` + RemovedCount int `json:"removed_count"` +} + +type userActivityToolAction struct { + AcceptedCount int `json:"accepted_count"` + RejectedCount int `json:"rejected_count"` +} + +type userActivityCCTools struct { + EditTool userActivityToolAction `json:"edit_tool"` + MultiEditTool userActivityToolAction `json:"multi_edit_tool"` + WriteTool userActivityToolAction `json:"write_tool"` + NotebookEditTool userActivityToolAction `json:"notebook_edit_tool"` +} + +// ── Console API response types (/v1/organizations/usage_report/claude_code) ─── + +// consoleUserActivityRecord is the JSON shape returned by /v1/organizations/usage_report/claude_code. +type consoleUserActivityRecord struct { + Date string `json:"date"` + Actor consoleActor `json:"actor"` + CoreMetrics consoleCoreMetrics `json:"core_metrics"` + ToolActions consoleToolActions `json:"tool_actions"` +} + +type consoleActor struct { + Type string `json:"type"` + EmailAddress string `json:"email_address"` + ApiKeyName string `json:"api_key_name"` +} + +type consoleCoreMetrics struct { + NumSessions int `json:"num_sessions"` + LinesOfCode consoleLinesOfCode `json:"lines_of_code"` + CommitsByClaudeCode int `json:"commits_by_claude_code"` + PullRequestsByClaudeCode int `json:"pull_requests_by_claude_code"` +} + +type consoleLinesOfCode struct { + Added int `json:"added"` + Removed int `json:"removed"` +} + +type consoleToolAction struct { + Accepted int `json:"accepted"` + Rejected int `json:"rejected"` +} + +type consoleToolActions struct { + EditTool consoleToolAction `json:"edit_tool"` + MultiEditTool consoleToolAction `json:"multi_edit_tool"` + WriteTool consoleToolAction `json:"write_tool"` + NotebookEditTool consoleToolAction `json:"notebook_edit_tool"` +} + +// ── Extractor ───────────────────────────────────────────────────────────────── + +// ExtractUserActivity parses raw user activity records into tool-layer tables. +func ExtractUserActivity(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*ClaudeCodeTaskData) + if !ok { + return errors.Default.New("task data is not ClaudeCodeTaskData") + } + connection := data.Connection + connection.Normalize() + + if strings.TrimSpace(connection.Organization) == "" { + taskCtx.GetLogger().Info("No organization configured, skipping user activity extraction") + return nil + } + + isConsole := connection.IsConsoleApiKey() + endpoint := "analytics/users" + if isConsole { + endpoint = "usage_report/claude_code" + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserActivityTable, + Options: claudeCodeRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: endpoint, + }, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + date, parseErr := parseAnalyticsDate(row.Input) + if parseErr != nil { + return nil, parseErr + } + if isConsole { + return extractConsoleUserActivity(data, date, row.Data) + } + return extractEnterpriseUserActivity(data, date, row.Data) + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func extractEnterpriseUserActivity(data *ClaudeCodeTaskData, date time.Time, raw []byte) ([]interface{}, errors.Error) { + var record userActivityRecord + if err := errors.Convert(json.Unmarshal(raw, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.User.Id) + if userId == "" { + userId = strings.TrimSpace(record.User.EmailAddress) + } + if userId == "" { + return nil, nil + } + + activity := &models.ClaudeCodeUserActivity{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + UserId: userId, + UserEmail: strings.TrimSpace(record.User.EmailAddress), + + ChatConversationCount: record.ChatMetrics.DistinctConversationCount, + ChatMessageCount: record.ChatMetrics.MessageCount, + ChatProjectsCreatedCount: record.ChatMetrics.DistinctProjectsCreatedCount, + ChatProjectsUsedCount: record.ChatMetrics.DistinctProjectsUsedCount, + ChatFilesUploadedCount: record.ChatMetrics.DistinctFilesUploadedCount, + ChatArtifactsCreatedCount: record.ChatMetrics.DistinctArtifactsCreatedCount, + ChatThinkingMessageCount: record.ChatMetrics.ThinkingMessageCount, + ChatSkillsUsedCount: record.ChatMetrics.DistinctSkillsUsedCount, + ChatConnectorsUsedCount: record.ChatMetrics.ConnectorsUsedCount, + + CCCommitCount: record.ClaudeCodeMetrics.CoreMetrics.CommitCount, + CCPullRequestCount: record.ClaudeCodeMetrics.CoreMetrics.PullRequestCount, + CCLinesAdded: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.AddedCount, + CCLinesRemoved: record.ClaudeCodeMetrics.CoreMetrics.LinesOfCode.RemovedCount, + CCSessionCount: record.ClaudeCodeMetrics.CoreMetrics.DistinctSessionCount, + + EditToolAccepted: record.ClaudeCodeMetrics.ToolActions.EditTool.AcceptedCount, + EditToolRejected: record.ClaudeCodeMetrics.ToolActions.EditTool.RejectedCount, + MultiEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.AcceptedCount, + MultiEditToolRejected: record.ClaudeCodeMetrics.ToolActions.MultiEditTool.RejectedCount, + WriteToolAccepted: record.ClaudeCodeMetrics.ToolActions.WriteTool.AcceptedCount, + WriteToolRejected: record.ClaudeCodeMetrics.ToolActions.WriteTool.RejectedCount, + NotebookEditToolAccepted: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.AcceptedCount, + NotebookEditToolRejected: record.ClaudeCodeMetrics.ToolActions.NotebookEditTool.RejectedCount, + + WebSearchCount: record.WebSearchCount, + } + return []interface{}{activity}, nil +} + +func extractConsoleUserActivity(data *ClaudeCodeTaskData, date time.Time, raw []byte) ([]interface{}, errors.Error) { + var record consoleUserActivityRecord + if err := errors.Convert(json.Unmarshal(raw, &record)); err != nil { + return nil, err + } + + userId := strings.TrimSpace(record.Actor.EmailAddress) + if userId == "" { + userId = strings.TrimSpace(record.Actor.ApiKeyName) + } + if userId == "" { + return nil, nil + } + + activity := &models.ClaudeCodeUserActivity{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Date: date, + UserId: userId, + UserEmail: strings.TrimSpace(record.Actor.EmailAddress), + + CCSessionCount: record.CoreMetrics.NumSessions, + CCCommitCount: record.CoreMetrics.CommitsByClaudeCode, + CCPullRequestCount: record.CoreMetrics.PullRequestsByClaudeCode, + CCLinesAdded: record.CoreMetrics.LinesOfCode.Added, + CCLinesRemoved: record.CoreMetrics.LinesOfCode.Removed, + + EditToolAccepted: record.ToolActions.EditTool.Accepted, + EditToolRejected: record.ToolActions.EditTool.Rejected, + MultiEditToolAccepted: record.ToolActions.MultiEditTool.Accepted, + MultiEditToolRejected: record.ToolActions.MultiEditTool.Rejected, + WriteToolAccepted: record.ToolActions.WriteTool.Accepted, + WriteToolRejected: record.ToolActions.WriteTool.Rejected, + NotebookEditToolAccepted: record.ToolActions.NotebookEditTool.Accepted, + NotebookEditToolRejected: record.ToolActions.NotebookEditTool.Rejected, + } + return []interface{}{activity}, nil +} + +// parseAnalyticsDate extracts the date from the raw row input JSON. +// The input is the claudeCodeDayInput or claudeCodeDateRangeInput encoded as JSON. +func parseAnalyticsDate(rawInput json.RawMessage) (time.Time, errors.Error) { + var dayInput claudeCodeDayInput + if err := json.Unmarshal(rawInput, &dayInput); err == nil && dayInput.Day != "" { + t, parseErr := time.Parse("2006-01-02", strings.TrimSpace(dayInput.Day)) + if parseErr == nil { + return utcDate(t), nil + } + } + var rangeInput claudeCodeDateRangeInput + if err := json.Unmarshal(rawInput, &rangeInput); err == nil && rangeInput.StartDate != "" { + t, parseErr := time.Parse("2006-01-02", strings.TrimSpace(rangeInput.StartDate)) + if parseErr == nil { + return utcDate(t), nil + } + } + return time.Time{}, errors.BadInput.New("could not parse date from raw input") +} diff --git a/backend/plugins/claude_code/tasks/user_activity_extractor_test.go b/backend/plugins/claude_code/tasks/user_activity_extractor_test.go new file mode 100644 index 00000000000..a2c1cdde488 --- /dev/null +++ b/backend/plugins/claude_code/tasks/user_activity_extractor_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestParseAnalyticsDateFromDayInput(t *testing.T) { + input, err := json.Marshal(claudeCodeDayInput{Day: "2026-03-04"}) + assert.NoError(t, err) + + date, parseErr := parseAnalyticsDate(input) + + assert.Nil(t, parseErr) + assert.Equal(t, time.Date(2026, 3, 4, 0, 0, 0, 0, time.UTC), date) +} + +func TestParseAnalyticsDateFromRangeInput(t *testing.T) { + input, err := json.Marshal(claudeCodeDateRangeInput{StartDate: "2026-03-04", EndDate: "2026-03-05"}) + assert.NoError(t, err) + + date, parseErr := parseAnalyticsDate(input) + + assert.Nil(t, parseErr) + assert.Equal(t, time.Date(2026, 3, 4, 0, 0, 0, 0, time.UTC), date) +} + +func TestParseAnalyticsDateReturnsErrorForInvalidInput(t *testing.T) { + date, parseErr := parseAnalyticsDate(json.RawMessage(`{"unknown":"2026-03-04"}`)) + + assert.NotNil(t, parseErr) + assert.True(t, date.IsZero()) +} diff --git a/backend/plugins/customize/e2e/import_test_case_execution_test.go b/backend/plugins/customize/e2e/import_test_case_execution_test.go index 92d5750b86b..a74c69940a6 100644 --- a/backend/plugins/customize/e2e/import_test_case_execution_test.go +++ b/backend/plugins/customize/e2e/import_test_case_execution_test.go @@ -71,6 +71,7 @@ func TestImportQaTestCaseExecutionsDataFlow(t *testing.T) { "start_time", "finish_time", "creator_id", + "is_invalid", "status", }) dataflowTester.VerifyTableWithRawData( @@ -104,6 +105,7 @@ func TestImportQaTestCaseExecutionsDataFlow(t *testing.T) { "start_time", "finish_time", "creator_id", + "is_invalid", "status", }) @@ -117,3 +119,51 @@ func TestImportQaTestCaseExecutionsDataFlow(t *testing.T) { }, ) } + +// TestImportQaTestCaseExecutions_NoIsInvalidColumn tests backward compatibility: +// Verifies that importing CSV files without the is_invalid column works correctly, +// and the is_invalid field defaults to false. +func TestImportQaTestCaseExecutions_NoIsInvalidColumn(t *testing.T) { + var plugin impl.Customize + dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin) + + // Flush the relevant table + dataflowTester.FlushTabler(&qa.QaTestCaseExecution{}) + dataflowTester.FlushTabler(&crossdomain.Account{}) + + // Create a new service instance + svc := service.NewService(dataflowTester.Dal) + + // Use the existing CSV file that does NOT contain is_invalid column + // This simulates backward compatibility with old CSV files + qaTestCaseExecutionsFile, err := os.Open("raw_tables/qa_test_case_executions_input.csv") + if err != nil { + t.Fatal(err) + } + defer qaTestCaseExecutionsFile.Close() + + // Define a dummy qaProjectId + qaProjectId := "test-backward-compat-project" + + // Import data from the CSV file (which has no is_invalid column) + err = svc.ImportQaTestCaseExecutions(qaProjectId, qaTestCaseExecutionsFile, false) + if err != nil { + t.Fatalf("ImportQaTestCaseExecutions failed: %v", err) + } + + // Verify the imported data has is_invalid defaulted to false + dataflowTester.VerifyTableWithRawData( + &qa.QaTestCaseExecution{}, + "snapshot_tables/qa_test_case_executions_output_no_is_invalid_column.csv", + []string{ + "id", + "qa_project_id", + "qa_test_case_id", + "create_time", + "start_time", + "finish_time", + "creator_id", + "is_invalid", + "status", + }) +} diff --git a/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output.csv b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output.csv index 1d51d4a0d66..cd78e9ec52e 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output.csv @@ -1,4 +1,4 @@ -id,qa_project_id,qa_test_case_id,create_time,start_time,finish_time,creator_id,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -exec-1,test-qa-project-id,tc-1,2023-03-01T10:00:00.000+00:00,2023-03-01T10:01:00.000+00:00,2023-03-01T10:05:00.000+00:00,csv:CsvAccount:0:user-a,SUCCESS,test-qa-project-id,,, -exec-2,test-qa-project-id,tc-2,2023-03-01T11:00:00.000+00:00,2023-03-01T11:02:00.000+00:00,2023-03-01T11:06:00.000+00:00,csv:CsvAccount:0:user-b,FAILED,test-qa-project-id,,, -exec-3,test-qa-project-id,tc-1,2023-03-02T10:00:00.000+00:00,2023-03-02T10:01:00.000+00:00,2023-03-02T10:04:00.000+00:00,csv:CsvAccount:0:user-a,SUCCESS,test-qa-project-id,,, +id,qa_project_id,qa_test_case_id,create_time,start_time,finish_time,creator_id,is_invalid,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +exec-1,test-qa-project-id,tc-1,2023-03-01T10:00:00.000+00:00,2023-03-01T10:01:00.000+00:00,2023-03-01T10:05:00.000+00:00,csv:CsvAccount:0:user-a,0,SUCCESS,test-qa-project-id,,, +exec-2,test-qa-project-id,tc-2,2023-03-01T11:00:00.000+00:00,2023-03-01T11:02:00.000+00:00,2023-03-01T11:06:00.000+00:00,csv:CsvAccount:0:user-b,0,FAILED,test-qa-project-id,,, +exec-3,test-qa-project-id,tc-1,2023-03-02T10:00:00.000+00:00,2023-03-02T10:01:00.000+00:00,2023-03-02T10:04:00.000+00:00,csv:CsvAccount:0:user-a,0,SUCCESS,test-qa-project-id,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_incremental.csv b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_incremental.csv index e214d734b23..90e496a27bd 100644 --- a/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_incremental.csv +++ b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_incremental.csv @@ -1,5 +1,5 @@ -id,qa_project_id,qa_test_case_id,create_time,start_time,finish_time,creator_id,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -exec-1,test-qa-project-id,tc-1,2023-03-04T10:00:00.000+00:00,2023-03-04T10:01:00.000+00:00,2023-03-04T10:06:00.000+00:00,csv:CsvAccount:0:user-a,FAILED,test-qa-project-id,,, -exec-2,test-qa-project-id,tc-2,2023-03-01T11:00:00.000+00:00,2023-03-01T11:02:00.000+00:00,2023-03-01T11:06:00.000+00:00,csv:CsvAccount:0:user-b,FAILED,test-qa-project-id,,, -exec-3,test-qa-project-id,tc-1,2023-03-02T10:00:00.000+00:00,2023-03-02T10:01:00.000+00:00,2023-03-02T10:04:00.000+00:00,csv:CsvAccount:0:user-a,SUCCESS,test-qa-project-id,,, -exec-4,test-qa-project-id,tc-3,2023-03-03T10:00:00.000+00:00,2023-03-03T10:01:00.000+00:00,2023-03-03T10:05:00.000+00:00,csv:CsvAccount:0:user-c,SUCCESS,test-qa-project-id,,, +id,qa_project_id,qa_test_case_id,create_time,start_time,finish_time,creator_id,is_invalid,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +exec-1,test-qa-project-id,tc-1,2023-03-04T10:00:00.000+00:00,2023-03-04T10:01:00.000+00:00,2023-03-04T10:06:00.000+00:00,csv:CsvAccount:0:user-a,0,FAILED,test-qa-project-id,,, +exec-2,test-qa-project-id,tc-2,2023-03-01T11:00:00.000+00:00,2023-03-01T11:02:00.000+00:00,2023-03-01T11:06:00.000+00:00,csv:CsvAccount:0:user-b,0,FAILED,test-qa-project-id,,, +exec-3,test-qa-project-id,tc-1,2023-03-02T10:00:00.000+00:00,2023-03-02T10:01:00.000+00:00,2023-03-02T10:04:00.000+00:00,csv:CsvAccount:0:user-a,0,SUCCESS,test-qa-project-id,,, +exec-4,test-qa-project-id,tc-3,2023-03-03T10:00:00.000+00:00,2023-03-03T10:01:00.000+00:00,2023-03-03T10:05:00.000+00:00,csv:CsvAccount:0:user-c,0,SUCCESS,test-qa-project-id,,, diff --git a/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_no_is_invalid_column.csv b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_no_is_invalid_column.csv new file mode 100644 index 00000000000..e313377a0c5 --- /dev/null +++ b/backend/plugins/customize/e2e/snapshot_tables/qa_test_case_executions_output_no_is_invalid_column.csv @@ -0,0 +1,4 @@ +id,qa_project_id,qa_test_case_id,create_time,start_time,finish_time,creator_id,is_invalid,status,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +exec-1,test-backward-compat-project,tc-1,2023-03-01T10:00:00.000+00:00,2023-03-01T10:01:00.000+00:00,2023-03-01T10:05:00.000+00:00,csv:CsvAccount:0:user-a,0,SUCCESS,test-backward-compat-project,,, +exec-2,test-backward-compat-project,tc-2,2023-03-01T11:00:00.000+00:00,2023-03-01T11:02:00.000+00:00,2023-03-01T11:06:00.000+00:00,csv:CsvAccount:0:user-b,0,FAILED,test-backward-compat-project,,, +exec-3,test-backward-compat-project,tc-1,2023-03-02T10:00:00.000+00:00,2023-03-02T10:01:00.000+00:00,2023-03-02T10:04:00.000+00:00,csv:CsvAccount:0:user-a,0,SUCCESS,test-backward-compat-project,,, \ No newline at end of file diff --git a/backend/plugins/customize/service/service.go b/backend/plugins/customize/service/service.go index 89e8833cf1c..8217be3a5c1 100644 --- a/backend/plugins/customize/service/service.go +++ b/backend/plugins/customize/service/service.go @@ -541,12 +541,16 @@ func (s *Service) qaTestCaseExecutionHandler(qaProjectId string) func(record map } delete(record, "creator_name") record["qa_project_id"] = qaProjectId + // Set default value for is_invalid if not present or empty in the CSV + if isInvalid, exists := record["is_invalid"]; !exists || isInvalid == "" { + record["is_invalid"] = false + } return s.dal.CreateWithMap(&qa.QaTestCaseExecution{}, record) } } // issueRepoCommitHandlerFactory returns a handler that will populate the `issue_commits` and `issue_repo_commits` table -// ths issueCommitsFields is used to filter the fields that should be inserted into the `issue_commits` table +// the issueCommitsFields is used to filter the fields that should be inserted into the `issue_commits` table func (s *Service) issueRepoCommitHandler(record map[string]interface{}) errors.Error { err := s.dal.CreateWithMap(&crossdomain.IssueRepoCommit{}, record) if err != nil { diff --git a/backend/plugins/dbt/dbt.go b/backend/plugins/dbt/dbt.go index ffb435ddc4d..02d055a9d8b 100644 --- a/backend/plugins/dbt/dbt.go +++ b/backend/plugins/dbt/dbt.go @@ -28,7 +28,6 @@ var PluginEntry impl.Dbt // standalone mode for debugging func main() { dbtCmd := &cobra.Command{Use: "dbt"} - _ = dbtCmd.MarkFlagRequired("projectPath") projectPath := dbtCmd.Flags().StringP("projectPath", "p", "/Users/abeizn/demoapp", "user dbt project directory.") projectGitURL := dbtCmd.Flags().StringP("projectGitURL", "g", "", "user dbt project git url.") projectName := dbtCmd.Flags().StringP("projectName", "n", "demoapp", "user dbt project name.") diff --git a/backend/plugins/dbt/impl/impl.go b/backend/plugins/dbt/impl/impl.go index 88eafa9f633..c984187ca10 100644 --- a/backend/plugins/dbt/impl/impl.go +++ b/backend/plugins/dbt/impl/impl.go @@ -18,6 +18,8 @@ limitations under the License. package impl import ( + "fmt" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -28,6 +30,7 @@ import ( var _ interface { plugin.PluginMeta plugin.PluginTask + plugin.CloseablePluginTask plugin.PluginModel } = (*Dbt)(nil) @@ -49,13 +52,14 @@ func (p Dbt) GetTablesInfo() []dal.Tabler { } func (p Dbt) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + taskCtx.GetLogger().Warn(nil, "The dbt plugin is deprecated and will be removed on August 31, 2026. Please migrate to alternative transformation approaches.") var op tasks.DbtOptions err := helper.Decode(options, &op, nil) if err != nil { return nil, err } - if op.ProjectPath == "" { - return nil, errors.Default.New("projectPath is required for dbt plugin") + if err := tasks.PrepareOptions(&op, taskCtx.GetConfig(tasks.DbtProjectBaseDirConfigKey)); err != nil { + return nil, err } if op.ProjectTarget == "" { @@ -71,6 +75,17 @@ func (p Dbt) RootPkgPath() string { return "github.com/apache/incubator-devlake/plugins/dbt" } +func (p Dbt) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.DbtTaskData) + if !ok || data == nil || data.Options == nil || !data.Options.ManagedProjectDir { + return nil + } + if err := tasks.CleanupManagedProjectDir(data.Options); err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("cleanup dbt project path %q", data.Options.ProjectPath)) + } + return nil +} + func (p Dbt) Name() string { return "dbt" } diff --git a/backend/plugins/dbt/tasks/convertor.go b/backend/plugins/dbt/tasks/convertor.go index 3d4ac56d385..a3a2d3f5611 100644 --- a/backend/plugins/dbt/tasks/convertor.go +++ b/backend/plugins/dbt/tasks/convertor.go @@ -234,4 +234,5 @@ var DbtConverterMeta = plugin.SubTaskMeta{ EntryPoint: DbtConverter, EnabledByDefault: true, Description: "Convert data by dbt", + Dependencies: []*plugin.SubTaskMeta{&GitMeta}, } diff --git a/backend/plugins/dbt/tasks/git.go b/backend/plugins/dbt/tasks/git.go index 49ea81f9098..b43f3efcf9d 100644 --- a/backend/plugins/dbt/tasks/git.go +++ b/backend/plugins/dbt/tasks/git.go @@ -32,20 +32,26 @@ func Git(taskCtx plugin.SubTaskContext) errors.Error { return nil } - // clean ProjectPath - err := os.RemoveAll(data.Options.ProjectPath) + projectBaseDir, err := ensureProjectBaseDir(data.Options.ProjectBaseDir) if err != nil { - logger.Error(err, "cleanup before clone dbt project failed") - return errors.Convert(err) + logger.Error(err, "prepare dbt workspace failed") + return err } + projectPath, mkErr := os.MkdirTemp(projectBaseDir, "project-*") + if mkErr != nil { + logger.Error(mkErr, "create managed dbt project directory failed") + return errors.Convert(mkErr) + } + data.Options.ProjectPath = projectPath - // git clone from ProjectGitURL into ProjectPath + // git clone from ProjectGitURL into a managed temporary project directory cmd := exec.Command("git", "clone", data.Options.ProjectGitURL, data.Options.ProjectPath) logger.Info("start clone dbt project: %v", cmd) - out, err := cmd.CombinedOutput() - if err != nil { - logger.Error(err, "clone dbt project failed") - return errors.Convert(err) + out, cloneErr := cmd.CombinedOutput() + if cloneErr != nil { + _ = os.RemoveAll(data.Options.ProjectPath) + logger.Error(cloneErr, "clone dbt project failed") + return errors.Convert(cloneErr) } logger.Info("clone dbt project success: %v", string(out)) return nil diff --git a/backend/plugins/dbt/tasks/options.go b/backend/plugins/dbt/tasks/options.go new file mode 100644 index 00000000000..496290d4318 --- /dev/null +++ b/backend/plugins/dbt/tasks/options.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/apache/incubator-devlake/core/errors" +) + +const ( + DbtProjectBaseDirConfigKey = "DBT_PROJECTS_DIR" + dbtProjectBaseDirName = "devlake-dbt-projects" +) + +func PrepareOptions(op *DbtOptions, configuredBaseDir string) errors.Error { + if op == nil { + return errors.Default.New("dbt options are required") + } + + baseDir, err := normalizeBaseDir(configuredBaseDir) + if err != nil { + return err + } + op.ProjectBaseDir = baseDir + op.ProjectPath = strings.TrimSpace(op.ProjectPath) + op.ProjectGitURL = strings.TrimSpace(op.ProjectGitURL) + + if op.ProjectGitURL != "" { + if err := validateProjectGitURL(op.ProjectGitURL); err != nil { + return err + } + op.ProjectPath = "" + op.ManagedProjectDir = true + return nil + } + + if op.ProjectPath == "" { + return errors.Default.New("projectPath is required for local dbt projects") + } + + projectPath, err := normalizePathWithinBase(baseDir, op.ProjectPath) + if err != nil { + return err + } + op.ProjectPath = projectPath + return nil +} + +func normalizeBaseDir(configuredBaseDir string) (string, errors.Error) { + baseDir := strings.TrimSpace(configuredBaseDir) + if baseDir == "" { + baseDir = filepath.Join(os.TempDir(), dbtProjectBaseDirName) + } + baseDir, err := filepath.Abs(filepath.Clean(baseDir)) + if err != nil { + return "", errors.Convert(err) + } + return baseDir, nil +} + +func normalizePathWithinBase(baseDir string, candidate string) (string, errors.Error) { + normalizedPath, err := filepath.Abs(filepath.Clean(candidate)) + if err != nil { + return "", errors.Convert(err) + } + rel, err := filepath.Rel(baseDir, normalizedPath) + if err != nil { + return "", errors.Convert(err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return "", errors.Default.New("projectPath must stay within " + baseDir) + } + return normalizedPath, nil +} + +func validateProjectGitURL(rawURL string) errors.Error { + u, err := url.Parse(rawURL) + if err != nil { + return errors.Convert(err) + } + if u.Scheme != "https" && u.Scheme != "ssh" { + return errors.Default.New("projectGitURL must use https:// or ssh://") + } + if u.Host == "" { + return errors.Default.New("projectGitURL must include a hostname") + } + if u.Path == "" || u.Path == "/" { + return errors.Default.New("projectGitURL must include a repository path") + } + return nil +} + +func ensureProjectBaseDir(baseDir string) (string, errors.Error) { + baseDir, err := normalizeBaseDir(baseDir) + if err != nil { + return "", err + } + if err := os.MkdirAll(baseDir, 0o755); err != nil { + return "", errors.Convert(err) + } + return baseDir, nil +} + +func CleanupManagedProjectDir(op *DbtOptions) errors.Error { + if op == nil || !op.ManagedProjectDir || op.ProjectPath == "" { + return nil + } + projectPath, err := normalizePathWithinBase(op.ProjectBaseDir, op.ProjectPath) + if err != nil { + return err + } + return errors.Convert(os.RemoveAll(projectPath)) +} diff --git a/backend/plugins/dbt/tasks/options_test.go b/backend/plugins/dbt/tasks/options_test.go new file mode 100644 index 00000000000..740e3527306 --- /dev/null +++ b/backend/plugins/dbt/tasks/options_test.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "path/filepath" + "testing" +) + +func TestPrepareOptionsNormalizesLocalProjectPath(t *testing.T) { + baseDir := t.TempDir() + projectDir := filepath.Join(baseDir, "nested", "..", "project") + + op := &DbtOptions{ProjectPath: projectDir} + if err := PrepareOptions(op, baseDir); err != nil { + t.Fatalf("PrepareOptions returned error: %v", err) + } + + want := filepath.Join(baseDir, "project") + if op.ProjectPath != want { + t.Fatalf("ProjectPath = %q, want %q", op.ProjectPath, want) + } + if op.ProjectBaseDir != baseDir { + t.Fatalf("ProjectBaseDir = %q, want %q", op.ProjectBaseDir, baseDir) + } + if op.ManagedProjectDir { + t.Fatal("ManagedProjectDir should be false for local projects") + } +} + +func TestPrepareOptionsRejectsPathTraversal(t *testing.T) { + baseDir := t.TempDir() + op := &DbtOptions{ProjectPath: filepath.Join(baseDir, "..", "outside")} + + if err := PrepareOptions(op, baseDir); err == nil { + t.Fatal("PrepareOptions should reject projectPath outside the configured base directory") + } +} + +func TestPrepareOptionsRejectsUnsafeGitURLs(t *testing.T) { + baseDir := t.TempDir() + cases := []string{ + "file:///tmp/evil", + "../relative/repo.git", + "git@github.com:apache/incubator-devlake.git", + } + + for _, rawURL := range cases { + t.Run(rawURL, func(t *testing.T) { + op := &DbtOptions{ProjectGitURL: rawURL} + if err := PrepareOptions(op, baseDir); err == nil { + t.Fatalf("PrepareOptions should reject %q", rawURL) + } + }) + } +} + +func TestPrepareOptionsAllowsManagedGitClone(t *testing.T) { + baseDir := t.TempDir() + op := &DbtOptions{ + ProjectPath: filepath.Join(baseDir, "ignored"), + ProjectGitURL: "https://github.com/apache/incubator-devlake.git", + } + + if err := PrepareOptions(op, baseDir); err != nil { + t.Fatalf("PrepareOptions returned error: %v", err) + } + if !op.ManagedProjectDir { + t.Fatal("ManagedProjectDir should be true for git-backed projects") + } + if op.ProjectPath != "" { + t.Fatalf("ProjectPath = %q, want empty for managed git clones", op.ProjectPath) + } +} + +func TestCleanupManagedProjectDirRemovesManagedPath(t *testing.T) { + baseDir := t.TempDir() + projectDir := filepath.Join(baseDir, "clone") + if err := os.MkdirAll(projectDir, 0o755); err != nil { + t.Fatalf("MkdirAll returned error: %v", err) + } + + op := &DbtOptions{ + ProjectBaseDir: baseDir, + ProjectPath: projectDir, + ManagedProjectDir: true, + } + if err := CleanupManagedProjectDir(op); err != nil { + t.Fatalf("CleanupManagedProjectDir returned error: %v", err) + } + if _, err := os.Stat(projectDir); !os.IsNotExist(err) { + t.Fatalf("expected %q to be removed, stat err = %v", projectDir, err) + } +} diff --git a/backend/plugins/dbt/tasks/task_data.go b/backend/plugins/dbt/tasks/task_data.go index d741954a9f1..00555fa2b90 100644 --- a/backend/plugins/dbt/tasks/task_data.go +++ b/backend/plugins/dbt/tasks/task_data.go @@ -39,6 +39,9 @@ type DbtOptions struct { // deprecated, dbt run args Args []string `json:"args"` Tasks []string `json:"tasks,omitempty"` + + ProjectBaseDir string `json:"-"` + ManagedProjectDir bool `json:"-"` } type DbtTaskData struct { diff --git a/backend/plugins/dora/api/data.go b/backend/plugins/dora/api/data.go index baada7a4c46..6f610ffa453 100644 --- a/backend/plugins/dora/api/data.go +++ b/backend/plugins/dora/api/data.go @@ -25,7 +25,7 @@ import ( const RAW_DEPLOYMENTS_TABLE = `dora_deplyments` -//TODO Please modify the folowing code to adapt to your plugin +//TODO Please modify the following code to adapt to your plugin /* POST /plugins/dora/deployments { @@ -39,7 +39,7 @@ func PostDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, const RAW_ISSUES_TABLE = `dora_issues` -//TODO Please modify the folowing code to adapt to your plugin +//TODO Please modify the following code to adapt to your plugin /* POST /plugins/dora/issues { @@ -51,7 +51,7 @@ func PostIssues(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil } -//TODO Please modify the folowing code to adapt to your plugin +//TODO Please modify the following code to adapt to your plugin /* POST /plugins/dora/issues/:id/close { diff --git a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv index 9517c252010..9d1e488f072 100644 --- a/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/cicd_deployment_commits.csv @@ -14,4 +14,5 @@ id,result,started_date,duration_sec,cicd_deployment_id,cicd_scope_id,repo_url,en 13,SUCCESS,2023-04-13T07:56:39.000+00:00,60,pipeline7,cicd2,REPO111,PRODUCTION,3,,commit13,2023-4-13 7:56:39,2023-04-13T07:57:39.000+00:00 14,FAILURE,2023-04-13T07:57:26.000+00:00,60,pipeline8,cicd3,REPO111,PRODUCTION,,,commit14,2023-4-13 7:57:26,2023-04-13T07:58:26.000+00:00 15,SUCCESS,2023-04-13T07:57:45.000+00:00,60,pipeline9,cicd3,REPO111,PRODUCTION,,,commit15,2023-4-13 7:57:45,2023-04-13T07:58:45.000+00:00 -16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00 \ No newline at end of file +16,SUCCESS,2023-04-13T07:58:24.000+00:00,60,pipeline10,cicd3,REPO333,,,,commit16,2023-4-13 7:58:24,2023-04-13T07:59:24.000+00:00 +17,SUCCESS,2023-04-13T07:59:00.000+00:00,60,pipeline11,cicd1,REPO111,PRODUCTION,,project1,direct_commit1,2023-4-13 7:59:00,2023-04-13T08:00:00.000+00:00 \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv index 97274eda1a5..f32370a330f 100644 --- a/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv +++ b/backend/plugins/dora/e2e/change_lead_time/project_pr_metrics.csv @@ -5,3 +5,4 @@ pr2,project1,2537845559d8db99e9cda6190f32b50ec979c722,,comment04,60000,3600000,5 pr3,project1,55f445997abbd5918da59d202d28762cd56fbd44,352980000,comment07,,345600000,6,,612180000,2023-04-07T04:51:47.000+00:00,2023-04-10T06:53:51.000+00:00,2023-04-11T06:53:51.000+00:00,2023-04-14T06:53:51.000+00:00,2023-04-13T07:30:34.000+00:00 pr4,project1,5ad0c09c447c19338f1dfbb65d89a3728962b3b7,702240000,comment10,90000000,,,,705840000,2023-04-05T04:51:47.000+00:00,2023-04-14T08:55:01.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, pr5,project1,62535543802631a0d3daf0b0b78c6a7e05e508fb,788640000,comment12,,18784080000,,,792240000,2023-04-04T04:51:47.000+00:00,2022-09-07T23:07:13.000+00:00,2023-04-13T07:55:01.000+00:00,2023-04-13T08:55:01.000+00:00, +pr7,project1,pr7_commit0,86400000,comment13,1800000,1800000,9,85980000,175980000,2023-04-11T07:00:00.000+00:00,2023-04-12T07:30:00.000+00:00,2023-04-12T07:00:00.000+00:00,2023-04-12T08:00:00.000+00:00,2023-04-13T07:52:26.000+00:00 diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv index 916a6f57d22..513e617b060 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_comments.csv @@ -12,3 +12,4 @@ comment09,pr3,2023-4-12 6:53:51,i comment10,pr4,2023-4-14 8:55:01,j comment11,pr4,2023-4-14 8:55:01,k comment12,pr5,2022-09-07 23:07:13,l +comment13,pr7,2023-4-12 7:30:00,m \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv index a6e112d9fef..e183d14cce3 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_request_commits.csv @@ -12,3 +12,4 @@ pr0_commit0,pr0,2022-1-10 4:51:47, 56b895f0443730c6d7abfbc51a05ab35abd2971f,pr4,2023-4-06 4:51:47, 5ad0c09c447c19338f1dfbb65d89a3728962b3b7,pr4,2023-4-05 4:51:47, 62535543802631a0d3daf0b0b78c6a7e05e508fb,pr5,2023-4-04 4:51:47, +pr7_commit0,pr7,2023-4-11 7:00:00, \ No newline at end of file diff --git a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv index 46a2050777d..3a3ec7ae9af 100644 --- a/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv +++ b/backend/plugins/dora/e2e/change_lead_time/pull_requests.csv @@ -6,3 +6,4 @@ pr3,repo1,a,pr_merge_commit3,2023-4-11 6:53:51,2023-4-14 6:53:51,deployment_comm pr4,repo1,,pr_merge_commit4,2023-4-13 7:55:01,2023-4-13 8:55:01,,, pr5,repo1,,pr_merge_commit5,2023-4-13 7:55:01,2023-4-13 8:55:01,,, pr6,repo1,,pr_merge_commit6,2023-4-13 7:55:01,,,, +pr7,repo1,a,commit9,2023-4-12 7:00:00,2023-4-12 8:00:00,,, diff --git a/backend/plugins/dora/tasks/change_lead_time_calculator.go b/backend/plugins/dora/tasks/change_lead_time_calculator.go index 3b3877b136a..3d5e2ffddec 100644 --- a/backend/plugins/dora/tasks/change_lead_time_calculator.go +++ b/backend/plugins/dora/tasks/change_lead_time_calculator.go @@ -279,47 +279,55 @@ func batchFetchFirstReviews(projectName string, db dal.Dal) (map[string]*code.Pu // batchFetchDeployments retrieves deployment commits for all merge commits in the given project. // Returns a map indexed by merge commit SHA for O(1) lookup performance. // -// The query finds the first successful production deployment for each merge commit by: -// 1. Finding deployment commits that have a previous successful deployment -// 2. Joining with commits_diffs to find which deployment included each merge commit -// 3. Filtering for successful production deployments -// 4. Ordering by started_date to get the earliest deployment +// Uses a two-phase strategy to avoid the "first deployment over-mapping" problem: // -// The map is indexed by merge_sha (from commits_diffs), not by deployment commit_sha, -// because the caller needs to look up deployments by PR merge_commit_sha. +// Phase 1 - Direct match: find successful PRODUCTION deployments whose commit_sha +// directly equals a PR's merge_commit_sha. Safe even for the very first deployment. +// +// Phase 2 - Diff-based fallback: use the commits_diffs join strategy, but deliberately +// skip the first deployment (prev_success_deployment_commit_id == "") to avoid over-mapping. func batchFetchDeployments(projectName string, db dal.Dal) (map[string]*devops.CicdDeploymentCommit, errors.Error) { - var results []*deploymentCommitWithMergeSha - - // Query finds the first deployment for each merge commit by using a window function - // to rank deployments by started_date, then filtering to keep only rank 1. + deploymentMap := make(map[string]*devops.CicdDeploymentCommit) + var directResults []*devops.CicdDeploymentCommit err := db.All( - &results, + &directResults, + dal.Select("dc.*"), + dal.From("cicd_deployment_commits dc"), + dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), + dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported + dal.Where("dc.result = ? AND pm.project_name = ?", devops.RESULT_SUCCESS, projectName), + dal.Orderby("dc.started_date ASC, dc.id ASC"), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to batch fetch direct deployments") + } + for _, dc := range directResults { + if _, exists := deploymentMap[dc.CommitSha]; !exists { + deploymentCopy := *dc + deploymentMap[dc.CommitSha] = &deploymentCopy + } + } + var diffResults []*deploymentCommitWithMergeSha + err = db.All( + &diffResults, dal.Select("dc.*, cd.commit_sha as merge_sha"), dal.From("cicd_deployment_commits dc"), dal.Join("LEFT JOIN cicd_deployment_commits p ON dc.prev_success_deployment_commit_id = p.id"), dal.Join("INNER JOIN commits_diffs cd ON cd.new_commit_sha = dc.commit_sha AND cd.old_commit_sha = COALESCE(p.commit_sha, '')"), dal.Join("LEFT JOIN project_mapping pm ON pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id"), dal.Where("dc.prev_success_deployment_commit_id <> ''"), - dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove this when multi-environment is supported + dal.Where("dc.environment = 'PRODUCTION'"), // TODO: remove when multi-environment is supported dal.Where("dc.result = ? AND pm.project_name = ?", devops.RESULT_SUCCESS, projectName), dal.Orderby("cd.commit_sha, dc.started_date ASC, dc.id ASC"), ) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to batch fetch deployments") + return nil, errors.Default.Wrap(err, "failed to batch fetch diff-based deployments") } - - // Build the map indexed by merge_sha for O(1) lookup. - // Keep only the first deployment for each merge commit (earliest by started_date). - deploymentMap := make(map[string]*devops.CicdDeploymentCommit, len(results)) - for _, result := range results { - // Only keep the first deployment for each merge_sha + for _, result := range diffResults { if _, exists := deploymentMap[result.MergeSha]; !exists { - // Copy the CicdDeploymentCommit without the MergeSha field deploymentCopy := result.CicdDeploymentCommit deploymentMap[result.MergeSha] = &deploymentCopy } } - return deploymentMap, nil } diff --git a/backend/plugins/gh-copilot/README.md b/backend/plugins/gh-copilot/README.md new file mode 100644 index 00000000000..1c4aeda2481 --- /dev/null +++ b/backend/plugins/gh-copilot/README.md @@ -0,0 +1,124 @@ + +# GitHub Copilot Plugin (Adoption Metrics) + +This plugin ingests GitHub Copilot **organization-level adoption metrics** (daily usage and seat assignments) and provides a Grafana dashboard for adoption trends. + +It follows the same structure/patterns as other DevLake data-source plugins (notably `backend/plugins/q_dev`). + +## What it collects + +**Phase 1 endpoints** (GitHub Copilot REST API): + +- `GET /orgs/{org}/copilot/billing` +- `GET /orgs/{org}/copilot/billing/seats` +- `GET /orgs/{org}/copilot/metrics` + +**Stored data (tool layer)**: + +- `_tool_copilot_org_metrics` (daily aggregates) +- `_tool_copilot_language_metrics` (editor/language breakdown) +- `_tool_copilot_seats` (seat assignments) + +## Data flow (high level) + +```mermaid +flowchart LR + GH[GitHub Copilot REST API] + RAW[(Raw tables\n_raw_copilot_*)] + TOOL[(Tool tables\n_tool_copilot_*)] + GRAF[Grafana Dashboard\nGitHub Copilot Adoption] + + GH --> RAW --> TOOL --> GRAF +``` + +## Repository layout + +- `api/` – REST layer for connections/scopes +- `impl/` – plugin meta, options, connection helpers +- `models/` – tool-layer models + migrations +- `tasks/` – collectors/extractors and pipeline registration +- `e2e/` – E2E fixtures and golden CSV assertions +- `docs/` – documentation assets + +## Setup + +### Prerequisites + +- GitHub Copilot Business or Enterprise enabled for the target organization +- A token that can access GitHub Copilot billing/metrics (classic PAT with `manage_billing:copilot` works) + +### 1) Create a connection + +1. DevLake UI → **Data Integrations → Add Connection → GitHub Copilot** +2. Fill in: + - **Name**: e.g. `GitHub Copilot Octodemo` + - **Endpoint**: defaults to `https://api.github.com` + - **Organization**: GitHub org slug + - **Token**: PAT with required scope +3. Click **Test Connection** (calls `GET /orgs/{org}/copilot/billing`). +4. Save the connection. + +### 2) Create a scope + +Add an organization scope for that connection. For Phase 1, `implementationDate` is optional. + +### 3) Create a blueprint (recipe) + +Use a blueprint plan like: + +```json +[ + [ + { + "plugin": "gh-copilot", + "options": { + "connectionId": 1, + "scopeId": "octodemo" + } + } + ] +] +``` + +Run the blueprint daily to keep metrics up to date. + +## Dashboard + +The Grafana dashboard JSON is in `grafana/dashboards/copilot/adoption.json`. + +Link: `grafana/dashboards/copilot/adoption.json` + +## Error handling guidance + +- **403 Forbidden** → token missing required billing/metrics scope, or org lacks GitHub Copilot access +- **404 Not Found** → incorrect org slug, or GitHub Copilot endpoints unavailable for the org +- **422 Unprocessable Entity** → GitHub Copilot metrics disabled in GitHub org settings + +- **429 Too Many Requests** → respect `Retry-After`; collectors implement backoff/retry + +Tokens are sanitized before persisting. When patching an existing connection, omit the token to retain the encrypted value already stored in DevLake. + +## Limitations (Phase 1) + +- Metrics endpoint is limited to a rolling **100-day** window (GitHub API constraint) +- GitHub enforces a privacy threshold (often **≥ 5 engaged users**) and may omit daily data +- Enterprise download endpoints and per-user metrics (JSONL exports) are intentionally deferred to Phase 2+ + +## More docs + +- Spec quickstart: `specs/001-copilot-metrics-plugin/quickstart.md` diff --git a/backend/plugins/gh-copilot/api/blueprint_v200.go b/backend/plugins/gh-copilot/api/blueprint_v200.go new file mode 100644 index 00000000000..97db93a5022 --- /dev/null +++ b/backend/plugins/gh-copilot/api/blueprint_v200.go @@ -0,0 +1,87 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + "github.com/apache/incubator-devlake/plugins/gh-copilot/tasks" +) + +// MakeDataSourcePipelinePlanV200 generates the pipeline plan for blueprint v2.0.0. +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + // load connection and scopes from the db + _, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + // map blueprint scopes (scopeId/scopeConfigId) to concrete stored scopes + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails) + if err != nil { + return nil, nil, err + } + + // Copilot metrics are org-level and currently don't map to a standard domain-layer top-level entity. + // Return an empty scope list to avoid adding meaningless project mappings. + return plan, nil, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.GhCopilotScope, models.GhCopilotScopeConfig], +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + task, err := helper.MakePipelinePlanTask( + "gh-copilot", + subtaskMetas, + nil, + tasks.GhCopilotOptions{ + ConnectionId: scope.ConnectionId, + ScopeId: scope.Id, + }, + ) + if err != nil { + return nil, err + } + + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} diff --git a/backend/plugins/gh-copilot/api/connection.go b/backend/plugins/gh-copilot/api/connection.go new file mode 100644 index 00000000000..fe02f16d0cb --- /dev/null +++ b/backend/plugins/gh-copilot/api/connection.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// PostConnections creates a new Copilot connection. +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.GhCopilotConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, err + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.GhCopilotConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.GhCopilotConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, err + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.GhCopilotConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.GhCopilotConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.GhCopilotConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +func validateConnection(connection *models.GhCopilotConnection) errors.Error { + if connection == nil { + return errors.BadInput.New("connection is required") + } + if connection.Organization == "" && !connection.HasEnterprise() { + return errors.BadInput.New("either enterprise or organization is required") + } + if connection.Token == "" { + return errors.BadInput.New("token is required") + } + if connection.RateLimitPerHour < 0 { + return errors.BadInput.New("rateLimitPerHour must be non-negative") + } + return nil +} diff --git a/backend/plugins/gh-copilot/api/connection_test.go b/backend/plugins/gh-copilot/api/connection_test.go new file mode 100644 index 00000000000..aa16ff6cfbf --- /dev/null +++ b/backend/plugins/gh-copilot/api/connection_test.go @@ -0,0 +1,91 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +func TestValidateConnection_Success(t *testing.T) { + connection := &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + Organization: "octodemo", + Token: "ghp_example", + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.NoError(t, err) +} + +func TestValidateConnection_MissingOrganization(t *testing.T) { + connection := &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + Token: "ghp_example", + }, + } + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "either enterprise or organization is required") +} + +func TestValidateConnection_EnterpriseOnly(t *testing.T) { + connection := &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + Enterprise: "my-enterprise", + Token: "ghp_example", + }, + } + connection.Normalize() + + err := validateConnection(connection) + assert.NoError(t, err) +} + +func TestValidateConnection_MissingToken(t *testing.T) { + connection := &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + Organization: "octodemo", + Token: "", + }, + } + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "token is required") +} + +func TestValidateConnection_InvalidRateLimit(t *testing.T) { + connection := &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + Organization: "octodemo", + Token: "ghp_example", + RateLimitPerHour: -1, + }, + } + + err := validateConnection(connection) + assert.Error(t, err) + assert.Contains(t, err.Error(), "rateLimitPerHour must be non-negative") +} diff --git a/backend/plugins/gh-copilot/api/init.go b/backend/plugins/gh-copilot/api/init.go new file mode 100644 index 00000000000..2cc920919df --- /dev/null +++ b/backend/plugins/gh-copilot/api/init.go @@ -0,0 +1,60 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +var ( + basicRes context.BasicRes + vld *validator.Validate + connectionHelper *helper.ConnectionApiHelper + dsHelper *helper.DsHelper[models.GhCopilotConnection, models.GhCopilotScope, models.GhCopilotScopeConfig] + raProxy *helper.DsRemoteApiProxyHelper[models.GhCopilotConnection] + raScopeList *helper.DsRemoteApiScopeListHelper[models.GhCopilotConnection, models.GhCopilotScope, GhCopilotRemotePagination] + raScopeSearch *helper.DsRemoteApiScopeSearchHelper[models.GhCopilotConnection, models.GhCopilotScope] +) + +// Init stores basic resources and configures shared helpers for API handlers. +func Init(br context.BasicRes, meta plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = helper.NewConnectionHelper(basicRes, vld, meta.Name()) + dsHelper = helper.NewDataSourceHelper[ + models.GhCopilotConnection, models.GhCopilotScope, models.GhCopilotScopeConfig, + ]( + basicRes, + meta.Name(), + []string{"id", "organization"}, + func(c models.GhCopilotConnection) models.GhCopilotConnection { + c.Normalize() + return c.Sanitize() + }, + func(s models.GhCopilotScope) models.GhCopilotScope { return s }, + nil, + ) + raProxy = helper.NewDsRemoteApiProxyHelper[models.GhCopilotConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = helper.NewDsRemoteApiScopeListHelper[models.GhCopilotConnection, models.GhCopilotScope, GhCopilotRemotePagination](raProxy, listGhCopilotRemoteScopes) + raScopeSearch = helper.NewDsRemoteApiScopeSearchHelper[models.GhCopilotConnection, models.GhCopilotScope](raProxy, searchGhCopilotRemoteScopes) +} diff --git a/backend/plugins/gh-copilot/api/remote_api.go b/backend/plugins/gh-copilot/api/remote_api.go new file mode 100644 index 00000000000..0ce2697f5ed --- /dev/null +++ b/backend/plugins/gh-copilot/api/remote_api.go @@ -0,0 +1,191 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// GhCopilotRemotePagination is a placeholder for scope list pagination. +// Copilot scopes are organization-level and currently return a single entry. +type GhCopilotRemotePagination struct { + Page int `json:"page"` +} + +func listGhCopilotRemoteScopes( + connection *models.GhCopilotConnection, + _ plugin.ApiClient, + _ string, + _ GhCopilotRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope], + nextPage *GhCopilotRemotePagination, + err errors.Error, +) { + if connection == nil { + return nil, nil, errors.BadInput.New("connection is required") + } + organization := strings.TrimSpace(connection.Organization) + + if connection.HasEnterprise() { + enterprise := strings.TrimSpace(connection.Enterprise) + if enterprise != "" { + scopeId := enterprise + if organization != "" { + scopeId = enterprise + "/" + organization + } + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: scopeId, + Name: scopeId, + FullName: scopeId, + Data: &models.GhCopilotScope{ + Id: scopeId, + Organization: organization, + Enterprise: enterprise, + Name: scopeId, + FullName: scopeId, + }, + }) + } + } else if organization != "" { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: organization, + Name: organization, + FullName: organization, + Data: &models.GhCopilotScope{ + Id: organization, + Organization: organization, + Name: organization, + FullName: organization, + }, + }) + } + + return children, nil, nil +} + +func searchGhCopilotRemoteScopes( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope], + err errors.Error, +) { + if params == nil { + return []dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope]{}, nil + } + query := strings.TrimSpace(params.Search) + if query == "" { + return []dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope]{}, nil + } + page := params.Page + pageSize := params.PageSize + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + + resp, err := apiClient.Get( + "user/orgs", + url.Values{ + "page": []string{strconv.Itoa(page)}, + "per_page": []string{strconv.Itoa(pageSize)}, + }, + nil, + ) + if err != nil { + return nil, err + } + + var orgs []struct { + Login string `json:"login"` + } + if err := api.UnmarshalResponse(resp, &orgs); err != nil { + return nil, err + } + + queryLower := strings.ToLower(query) + for _, org := range orgs { + orgName := strings.TrimSpace(org.Login) + if orgName == "" { + continue + } + if !strings.Contains(strings.ToLower(orgName), queryLower) { + continue + } + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.GhCopilotScope]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: orgName, + Name: orgName, + FullName: orgName, + Data: &models.GhCopilotScope{ + Id: orgName, + Organization: orgName, + Name: orgName, + FullName: orgName, + }, + }) + } + + return children, nil +} + +// RemoteScopes list all available scopes (organizations) for this connection +// @Summary list all available scopes (organizations) for this connection +// @Description list all available scopes (organizations) for this connection +// @Tags plugins/gh-copilot +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.GhCopilotScope] +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/gh-copilot/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// SearchRemoteScopes searches organization scopes for this connection +// @Summary searches organization scopes for this connection +// @Description searches organization scopes for this connection +// @Tags plugins/gh-copilot +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.GhCopilotScope] "the parentIds are always null" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/gh-copilot/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} diff --git a/backend/plugins/gh-copilot/api/scope.go b/backend/plugins/gh-copilot/api/scope.go new file mode 100644 index 00000000000..dbd8cca59b1 --- /dev/null +++ b/backend/plugins/gh-copilot/api/scope.go @@ -0,0 +1,59 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +type PutScopesReqBody = helper.PutScopesReqBody[models.GhCopilotScope] +type ScopeDetail = srvhelper.ScopeDetail[models.GhCopilotScope, models.GhCopilotScopeConfig] + +// PutScopes creates or updates Copilot organization scopes. +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScopeList retrieves scopes for a connection with optional pagination. +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope returns the scope detail for a given scope ID. +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope updates a scope record. +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope and optionally associated data. +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState returns the latest sync state for a scope. +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/gh-copilot/api/scope_config.go b/backend/plugins/gh-copilot/api/scope_config.go new file mode 100644 index 00000000000..7184652191d --- /dev/null +++ b/backend/plugins/gh-copilot/api/scope_config.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig creates a new scope configuration +// @Summary Create scope config +// @Description Create scope config for GitHub Copilot +// @Tags plugins/gh-copilot +// @Accept json +// @Param connectionId path int true "connection ID" +// @Param request body models.GhCopilotScopeConfig true "scope config" +// @Success 200 {object} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// GetScopeConfigList returns all scope configurations for a connection +// @Summary Get scope configs +// @Description Get all scope configs for a connection +// @Tags plugins/gh-copilot +// @Param connectionId path int true "connection ID" +// @Success 200 {array} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// GetScopeConfig returns a scope configuration by id +// @Summary Get scope config +// @Description Get a scope config by ID +// @Tags plugins/gh-copilot +// @Param connectionId path int true "connection ID" +// @Param scopeConfigId path int true "scope config ID" +// @Success 200 {object} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// PatchScopeConfig updates a scope configuration +// @Summary Patch scope config +// @Description Update a scope config +// @Tags plugins/gh-copilot +// @Accept json +// @Param connectionId path int true "connection ID" +// @Param scopeConfigId path int true "scope config ID" +// @Param request body models.GhCopilotScopeConfig true "scope config" +// @Success 200 {object} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// DeleteScopeConfig deletes a scope configuration +// @Summary Delete scope config +// @Description Delete a scope config +// @Tags plugins/gh-copilot +// @Param connectionId path int true "connection ID" +// @Param scopeConfigId path int true "scope config ID" +// @Success 200 {object} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig returns projects related to a scope config +// @Summary Get projects by scope config +// @Description Get projects details related by scope config +// @Tags plugins/gh-copilot +// @Param scopeConfigId path int true "scope config ID" +// @Success 200 {object} models.GhCopilotScopeConfig +// @Failure 400 {object} shared.ApiBody "bad request" +// @Failure 500 {object} shared.ApiBody "internal error" +// @Router /plugins/gh-copilot/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/gh-copilot/api/test_connection.go b/backend/plugins/gh-copilot/api/test_connection.go new file mode 100644 index 00000000000..3dedc859a67 --- /dev/null +++ b/backend/plugins/gh-copilot/api/test_connection.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + "github.com/apache/incubator-devlake/plugins/gh-copilot/service" +) + +// TestConnection validates a Copilot connection before saving it. +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.GhCopilotConnection{} + if err := helper.Decode(input.Body, connection, vld); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(context.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection validates a stored Copilot connection with optional overrides. +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.GhCopilotConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, errors.BadInput.Wrap(err, "find connection from db")) + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + connection.Normalize() + if err := validateConnection(connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + + result, err := service.TestConnection(context.Background(), basicRes, connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/gh-copilot/copilot.go b/backend/plugins/gh-copilot/copilot.go new file mode 100644 index 00000000000..c4de27e9f04 --- /dev/null +++ b/backend/plugins/gh-copilot/copilot.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.GhCopilot + +// standalone mode for debugging collectors. +func main() { + cmd := &cobra.Command{Use: "gh-copilot"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "gh-copilot connection id") + scopeId := cmd.Flags().StringP("scopeId", "s", "", "gh-copilot scope id (organization)") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data created after the specified time") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("scopeId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "scopeId": *scopeId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/gh-copilot/e2e/metrics/language_breakdown.csv b/backend/plugins/gh-copilot/e2e/metrics/language_breakdown.csv new file mode 100644 index 00000000000..6e522f2bb03 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/language_breakdown.csv @@ -0,0 +1,5 @@ +connection_id,scope_id,date,editor,language,engaged_users,suggestions,acceptances,lines_suggested,lines_accepted +1,octodemo,2025-09-01T00:00:00.000+00:00,vscode,python,6,100,20,300,60 +1,octodemo,2025-09-01T00:00:00.000+00:00,jetbrains,ruby,4,50,10,120,30 +1,octodemo,2025-09-02T00:00:00.000+00:00,vscode,python,8,180,36,450,180 +1,octodemo,2025-09-02T00:00:00.000+00:00,jetbrains,go,2,20,4,50,20 diff --git a/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_metrics.csv b/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_metrics.csv new file mode 100644 index 00000000000..7bde72fcc49 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_metrics.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""octodemo"",""Organization"":""octodemo"",""Endpoint"":""https://api.github.com""}","{""day"":""2025-09-01"",""daily_active_users"":10,""totals_by_language_feature"":[{""language"":""python"",""feature"":""code"",""code_generation_activity_count"":100,""code_acceptance_activity_count"":20,""loc_suggested_to_add_sum"":300,""loc_suggested_to_delete_sum"":0,""loc_added_sum"":60,""loc_deleted_sum"":0},{""language"":""ruby"",""feature"":""code"",""code_generation_activity_count"":50,""code_acceptance_activity_count"":10,""loc_suggested_to_add_sum"":120,""loc_suggested_to_delete_sum"":0,""loc_added_sum"":30,""loc_deleted_sum"":0}]}",https://api.github.com/orgs/octodemo/copilot/metrics/reports/organization-1-day?day=2025-09-01,null,2025-09-03 00:00:00.000 +2,"{""ConnectionId"":1,""ScopeId"":""octodemo"",""Organization"":""octodemo"",""Endpoint"":""https://api.github.com""}","{""day"":""2025-09-02"",""daily_active_users"":12,""totals_by_language_feature"":[{""language"":""python"",""feature"":""code"",""code_generation_activity_count"":180,""code_acceptance_activity_count"":36,""loc_suggested_to_add_sum"":450,""loc_suggested_to_delete_sum"":0,""loc_added_sum"":180,""loc_deleted_sum"":0},{""language"":""go"",""feature"":""code"",""code_generation_activity_count"":20,""code_acceptance_activity_count"":4,""loc_suggested_to_add_sum"":50,""loc_suggested_to_delete_sum"":0,""loc_added_sum"":20,""loc_deleted_sum"":0}]}",https://api.github.com/orgs/octodemo/copilot/metrics/reports/organization-1-day?day=2025-09-02,null,2025-09-03 00:00:00.000 diff --git a/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_seats.csv b/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_seats.csv new file mode 100644 index 00000000000..efefd419869 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/raw_tables/_raw_copilot_seats.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""octodemo"",""Organization"":""octodemo"",""Endpoint"":""https://api.github.com""}","{""created_at"":""2023-08-29T02:50:42+03:00"",""assignee"":{""login"":""nathos"",""id"":4215,""type"":""User""},""pending_cancellation_date"":null,""plan_type"":""enterprise"",""last_authenticated_at"":""2025-12-04T15:53:22Z"",""updated_at"":""2024-02-01T03:00:00+03:00"",""last_activity_at"":""2025-11-06T19:12:15+03:00"",""last_activity_editor"":""copilot_pr_review""}",https://api.github.com/orgs/octodemo/copilot/billing/seats,null,2025-09-03 00:00:00.000 +2,"{""ConnectionId"":1,""ScopeId"":""octodemo"",""Organization"":""octodemo"",""Endpoint"":""https://api.github.com""}","{""created_at"":""2024-01-10T10:11:12Z"",""assignee"":{""login"":""octocat"",""id"":1,""type"":""User""},""pending_cancellation_date"":null,""plan_type"":""enterprise"",""last_authenticated_at"":null,""updated_at"":""2024-02-02T00:00:00Z"",""last_activity_at"":null,""last_activity_editor"":""vscode/1.0.0/copilot-chat/0.1.0""}",https://api.github.com/orgs/octodemo/copilot/billing/seats,null,2025-09-03 00:00:00.000 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv new file mode 100644 index 00000000000..7a74a5cc81d --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv @@ -0,0 +1,3 @@ +connection_id,scope_id,day,enterprise_id,daily_active_users,weekly_active_users,monthly_active_users,monthly_active_chat_users,monthly_active_agent_users,daily_active_cli_users,daily_active_copilot_code_review_users,daily_passive_copilot_code_review_users,weekly_active_copilot_code_review_users,weekly_passive_copilot_code_review_users,monthly_active_copilot_code_review_users,monthly_passive_copilot_code_review_users,chat_panel_agent_mode,chat_panel_ask_mode,chat_panel_custom_mode,chat_panel_edit_mode,chat_panel_plan_mode,chat_panel_unknown_mode,pr_total_reviewed,pr_total_created,pr_total_merged,pr_median_minutes_to_merge,pr_total_suggestions,pr_total_applied_suggestions,pr_total_created_by_copilot,pr_total_reviewed_by_copilot,pr_total_merged_created_by_copilot,pr_total_merged_reviewed_by_copilot,pr_median_min_to_merge_copilot_authored,pr_median_min_to_merge_copilot_reviewed,pr_total_copilot_suggestions,pr_total_copilot_applied_suggestions,user_initiated_interaction_count,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum,cli_session_count,cli_request_count,cli_prompt_count,cli_output_token_sum,cli_prompt_token_sum +1,octodemo,2025-09-01T00:00:00.000+00:00,,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,octodemo,2025-09-02T00:00:00.000+00:00,,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_metrics_by_language_feature.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_metrics_by_language_feature.csv new file mode 100644 index 00000000000..8cf3cd6e045 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_metrics_by_language_feature.csv @@ -0,0 +1,5 @@ +connection_id,scope_id,day,language,feature,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum +1,octodemo,2025-09-01T00:00:00.000+00:00,python,code,100,20,300,0,60,0 +1,octodemo,2025-09-01T00:00:00.000+00:00,ruby,code,50,10,120,0,30,0 +1,octodemo,2025-09-02T00:00:00.000+00:00,python,code,180,36,450,0,180,0 +1,octodemo,2025-09-02T00:00:00.000+00:00,go,code,20,4,50,0,20,0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_metrics.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_metrics.csv new file mode 100644 index 00000000000..8986efcb879 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_org_metrics.csv @@ -0,0 +1,3 @@ +connection_id,scope_id,date,total_active_users,total_engaged_users,completion_suggestions,completion_acceptances,completion_lines_suggested,completion_lines_accepted,ide_chats,ide_chat_copy_events,ide_chat_insertion_events,ide_chat_engaged_users,dotcom_chats,dotcom_chat_engaged_users,seat_active_count,seat_total +1,octodemo,2025-09-01T00:00:00.000+00:00,10,8,150,30,420,90,8,3,1,3,5,2,1,2 +1,octodemo,2025-09-02T00:00:00.000+00:00,12,9,200,40,500,200,10,4,2,4,2,1,1,2 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv new file mode 100644 index 00000000000..291a98ca607 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv @@ -0,0 +1,3 @@ +connection_id,organization,user_login,user_id,user_name,user_email,plan_type,assigning_team_id,assigning_team_name,assigning_team_slug,created_at,last_activity_at,last_activity_editor,last_authenticated_at,pending_cancellation_date,updated_at +1,octodemo,nathos,4215,,,enterprise,0,,,2023-08-28T23:50:42.000+00:00,2025-11-06T16:12:15.000+00:00,copilot_pr_review,2025-12-04T15:53:22.000+00:00,,2024-02-01T00:00:00.000+00:00 +1,octodemo,octocat,1,,,enterprise,0,,,2024-01-10T10:11:12.000+00:00,,vscode/1.0.0/copilot-chat/0.1.0,,,2024-02-02T00:00:00.000+00:00 diff --git a/backend/plugins/gh-copilot/e2e/metrics_test.go b/backend/plugins/gh-copilot/e2e/metrics_test.go new file mode 100644 index 00000000000..2f339f95d16 --- /dev/null +++ b/backend/plugins/gh-copilot/e2e/metrics_test.go @@ -0,0 +1,87 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + "time" + + "github.com/apache/incubator-devlake/core/config" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + "github.com/apache/incubator-devlake/plugins/gh-copilot/tasks" +) + +func TestCopilotMetricsDataFlow(t *testing.T) { + cfg := config.GetConfig() + dbUrl := cfg.GetString("E2E_DB_URL") + if dbUrl == "" { + t.Skip("skipping e2e test: E2E_DB_URL is not set") + } + if err := runner.CheckDbConnection(dbUrl, 10*time.Second); err != nil { + t.Skipf("skipping e2e test: cannot connect to E2E_DB_URL: %v", err) + } + + var copilot impl.GhCopilot + dataflowTester := e2ehelper.NewDataFlowTester(t, "gh-copilot", copilot) + + taskData := &tasks.GhCopilotTaskData{ + Options: &tasks.GhCopilotOptions{ + ConnectionId: 1, + ScopeId: "octodemo", + }, + Connection: &models.GhCopilotConnection{ + GhCopilotConn: models.GhCopilotConn{ + RestConnection: helper.RestConnection{Endpoint: "https://api.github.com"}, + Organization: "octodemo", + RateLimitPerHour: 5000, + }, + }, + } + + dataflowTester.ImportCsvIntoRawTable("./metrics/raw_tables/_raw_copilot_metrics.csv", "_raw_copilot_org_metrics") + dataflowTester.ImportCsvIntoRawTable("./metrics/raw_tables/_raw_copilot_seats.csv", "_raw_copilot_seats") + + dataflowTester.FlushTabler(&models.GhCopilotSeat{}) + dataflowTester.FlushTabler(&models.GhCopilotEnterpriseDailyMetrics{}) + dataflowTester.FlushTabler(&models.GhCopilotMetricsByLanguageFeature{}) + + dataflowTester.Subtask(tasks.ExtractSeatsMeta, taskData) + dataflowTester.Subtask(tasks.ExtractOrgMetricsMeta, taskData) + + dataflowTester.VerifyTableWithOptions(&models.GhCopilotEnterpriseDailyMetrics{}, e2ehelper.TableOptions{ + CSVRelPath: "./metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.VerifyTableWithOptions(&models.GhCopilotSeat{}, e2ehelper.TableOptions{ + CSVRelPath: "./metrics/snapshot_tables/_tool_copilot_seats.csv", + IgnoreTypes: []interface{}{ + common.RawDataOrigin{}, + }, + }) + + dataflowTester.VerifyTableWithOptions(&models.GhCopilotMetricsByLanguageFeature{}, e2ehelper.TableOptions{ + CSVRelPath: "./metrics/snapshot_tables/_tool_copilot_metrics_by_language_feature.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/gh-copilot/impl/connection_helper.go b/backend/plugins/gh-copilot/impl/connection_helper.go new file mode 100644 index 00000000000..d94f9947906 --- /dev/null +++ b/backend/plugins/gh-copilot/impl/connection_helper.go @@ -0,0 +1,20 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +// Deprecated stub file: connection testing helpers now reside in the service package. diff --git a/backend/plugins/gh-copilot/impl/connection_helper_test.go b/backend/plugins/gh-copilot/impl/connection_helper_test.go new file mode 100644 index 00000000000..713aec48b7a --- /dev/null +++ b/backend/plugins/gh-copilot/impl/connection_helper_test.go @@ -0,0 +1,20 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +// Legacy placeholder: connection helper tests moved to the service package. diff --git a/backend/plugins/gh-copilot/impl/impl.go b/backend/plugins/gh-copilot/impl/impl.go new file mode 100644 index 00000000000..aefcd1ba6a4 --- /dev/null +++ b/backend/plugins/gh-copilot/impl/impl.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/gh-copilot/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.DataSourcePluginBlueprintV200 + plugin.PluginMigration + plugin.CloseablePluginTask +} = (*GhCopilot)(nil) + +// GhCopilot is the plugin entrypoint implementing DevLake interfaces. +type GhCopilot struct{} + +func (p GhCopilot) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p GhCopilot) Description() string { + return "Collect GitHub Copilot usage metrics (enterprise and organization level)" +} + +func (p GhCopilot) Name() string { + return "gh-copilot" +} + +func (p GhCopilot) Connection() dal.Tabler { + return &models.GhCopilotConnection{} +} + +func (p GhCopilot) Scope() plugin.ToolLayerScope { + return &models.GhCopilotScope{} +} + +func (p GhCopilot) ScopeConfig() dal.Tabler { + return &models.GhCopilotScopeConfig{} +} + +func (p GhCopilot) GetTablesInfo() []dal.Tabler { + return models.GetTablesInfo() +} + +func (p GhCopilot) SubTaskMetas() []plugin.SubTaskMeta { + return tasks.GetSubTaskMetas() +} + +func (p GhCopilot) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.GhCopilotOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.GhCopilotConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + NormalizeConnection(connection) + + taskData := &tasks.GhCopilotTaskData{ + Options: &op, + Connection: connection, + } + + return taskData, nil +} + +func (p GhCopilot) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "GET": api.GetScopeConfig, + "PATCH": api.PatchScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p GhCopilot) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p GhCopilot) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/gh-copilot" +} + +func (p GhCopilot) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p GhCopilot) Close(taskCtx plugin.TaskContext) errors.Error { + return nil +} diff --git a/backend/plugins/gh-copilot/impl/options.go b/backend/plugins/gh-copilot/impl/options.go new file mode 100644 index 00000000000..015196c8a80 --- /dev/null +++ b/backend/plugins/gh-copilot/impl/options.go @@ -0,0 +1,25 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + +// NormalizeConnection ensures required defaults are set before use. +func NormalizeConnection(connection *models.GhCopilotConnection) { + connection.Normalize() +} diff --git a/backend/plugins/gh-copilot/models/connection.go b/backend/plugins/gh-copilot/models/connection.go new file mode 100644 index 00000000000..676d06b4f85 --- /dev/null +++ b/backend/plugins/gh-copilot/models/connection.go @@ -0,0 +1,121 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const ( + // DefaultEndpoint is the GitHub REST API endpoint used for Copilot metrics. + DefaultEndpoint = "https://api.github.com" + // DefaultRateLimitPerHour mirrors GitHub's default rate limit for PATs. + DefaultRateLimitPerHour = 5000 +) + +// GhCopilotConn stores GitHub Copilot connection settings. +type GhCopilotConn struct { + helper.RestConnection `mapstructure:",squash"` + + Token string `mapstructure:"token" json:"token"` + Organization string `mapstructure:"organization" json:"organization"` + Enterprise string `mapstructure:"enterprise" json:"enterprise,omitempty" gorm:"type:varchar(100)"` + RateLimitPerHour int `mapstructure:"rateLimitPerHour" json:"rateLimitPerHour"` +} + +// HasEnterprise returns true if the connection is configured for enterprise-level access. +func (conn *GhCopilotConn) HasEnterprise() bool { + return conn != nil && strings.TrimSpace(conn.Enterprise) != "" +} + +// SetupAuthentication implements plugin.ApiAuthenticator so helper.NewApiClientFromConnection +// can attach the Authorization header for GitHub API requests. +func (conn *GhCopilotConn) SetupAuthentication(request *http.Request) errors.Error { + if conn == nil { + return errors.BadInput.New("connection is required") + } + token := strings.TrimSpace(conn.Token) + if token == "" { + return errors.BadInput.New("token is required") + } + + lower := strings.ToLower(token) + if strings.HasPrefix(lower, "bearer ") || strings.HasPrefix(lower, "token ") { + request.Header.Set("Authorization", token) + return nil + } + request.Header.Set("Authorization", "Bearer "+token) + return nil +} + +func (conn *GhCopilotConn) Sanitize() GhCopilotConn { + if conn == nil { + return GhCopilotConn{} + } + clone := *conn + clone.Token = utils.SanitizeString(clone.Token) + return clone +} + +// GhCopilotConnection persists connection details with metadata required by DevLake. +type GhCopilotConnection struct { + helper.BaseConnection `mapstructure:",squash"` + GhCopilotConn `mapstructure:",squash"` +} + +func (GhCopilotConnection) TableName() string { + return "_tool_copilot_connections" +} + +func (connection GhCopilotConnection) Sanitize() GhCopilotConnection { + connection.GhCopilotConn = connection.GhCopilotConn.Sanitize() + return connection +} + +func (connection *GhCopilotConnection) MergeFromRequest(target *GhCopilotConnection, body map[string]interface{}) error { + if target == nil { + return nil + } + originalToken := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + sanitizedOriginal := utils.SanitizeString(originalToken) + if target.Token == "" || target.Token == sanitizedOriginal { + target.Token = originalToken + } + return nil +} + +// Normalize applies default connection values where necessary. +func (connection *GhCopilotConnection) Normalize() { + if connection == nil { + return + } + if connection.Endpoint == "" { + connection.Endpoint = DefaultEndpoint + } + if connection.RateLimitPerHour <= 0 { + connection.RateLimitPerHour = DefaultRateLimitPerHour + } +} diff --git a/backend/plugins/gh-copilot/models/connection_auth_test.go b/backend/plugins/gh-copilot/models/connection_auth_test.go new file mode 100644 index 00000000000..3403f43fe7b --- /dev/null +++ b/backend/plugins/gh-copilot/models/connection_auth_test.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGhCopilotConn_SetupAuthentication_BearerPrefix(t *testing.T) { + conn := &GhCopilotConn{Token: "Bearer abc"} + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + + err2 := conn.SetupAuthentication(req) + require.NoError(t, err2) + require.Equal(t, "Bearer abc", req.Header.Get("Authorization")) +} + +func TestGhCopilotConn_SetupAuthentication_TokenPrefix(t *testing.T) { + conn := &GhCopilotConn{Token: "token abc"} + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + + err2 := conn.SetupAuthentication(req) + require.NoError(t, err2) + require.Equal(t, "token abc", req.Header.Get("Authorization")) +} + +func TestGhCopilotConn_SetupAuthentication_RawToken(t *testing.T) { + conn := &GhCopilotConn{Token: "abc"} + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + + err2 := conn.SetupAuthentication(req) + require.NoError(t, err2) + require.Equal(t, "Bearer abc", req.Header.Get("Authorization")) +} + +func TestGhCopilotConn_SetupAuthentication_TrimsWhitespace(t *testing.T) { + conn := &GhCopilotConn{Token: " abc "} + req, err := http.NewRequest(http.MethodGet, "https://example.com", nil) + require.NoError(t, err) + + err2 := conn.SetupAuthentication(req) + require.NoError(t, err2) + require.Equal(t, "Bearer abc", req.Header.Get("Authorization")) +} diff --git a/backend/plugins/gh-copilot/models/enterprise_metrics.go b/backend/plugins/gh-copilot/models/enterprise_metrics.go new file mode 100644 index 00000000000..967e3ecd326 --- /dev/null +++ b/backend/plugins/gh-copilot/models/enterprise_metrics.go @@ -0,0 +1,189 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// CopilotActivityMetrics contains the common activity/LOC fields shared across all breakdown tables. +type CopilotActivityMetrics struct { + UserInitiatedInteractionCount int `json:"userInitiatedInteractionCount" gorm:"comment:Chat messages and inline prompts initiated by user"` + CodeGenerationActivityCount int `json:"codeGenerationActivityCount" gorm:"comment:Number of code suggestions/generations made"` + CodeAcceptanceActivityCount int `json:"codeAcceptanceActivityCount" gorm:"comment:Number of suggestions accepted by user"` + LocSuggestedToAddSum int `json:"locSuggestedToAddSum" gorm:"comment:Lines of code suggested for addition"` + LocSuggestedToDeleteSum int `json:"locSuggestedToDeleteSum" gorm:"comment:Lines of code suggested for deletion"` + LocAddedSum int `json:"locAddedSum" gorm:"comment:Lines of code actually added (accepted)"` + LocDeletedSum int `json:"locDeletedSum" gorm:"comment:Lines of code actually deleted (accepted)"` +} + +// CopilotCodeMetrics contains code generation/acceptance metrics without user interaction count. +type CopilotCodeMetrics struct { + CodeGenerationActivityCount int `json:"codeGenerationActivityCount"` + CodeAcceptanceActivityCount int `json:"codeAcceptanceActivityCount"` + LocSuggestedToAddSum int `json:"locSuggestedToAddSum"` + LocSuggestedToDeleteSum int `json:"locSuggestedToDeleteSum"` + LocAddedSum int `json:"locAddedSum"` + LocDeletedSum int `json:"locDeletedSum"` +} + +// CopilotCliMetrics contains CLI usage breakdown metrics. +type CopilotCliMetrics struct { + CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"` + CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"` + CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"` + CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"` + CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"` +} + +// GhCopilotEnterpriseDailyMetrics captures daily enterprise-level aggregate Copilot metrics. +type GhCopilotEnterpriseDailyMetrics struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + DailyActiveUsers int `json:"dailyActiveUsers"` + WeeklyActiveUsers int `json:"weeklyActiveUsers"` + MonthlyActiveUsers int `json:"monthlyActiveUsers"` + MonthlyActiveChatUsers int `json:"monthlyActiveChatUsers"` + MonthlyActiveAgentUsers int `json:"monthlyActiveAgentUsers"` + + // CLI active users + DailyActiveCliUsers int `json:"dailyActiveCliUsers" gorm:"comment:Daily active CLI users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"dailyActiveCopilotCodeReviewUsers"` + DailyPassiveCopilotCodeReviewUsers int `json:"dailyPassiveCopilotCodeReviewUsers"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weeklyActiveCopilotCodeReviewUsers"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weeklyPassiveCopilotCodeReviewUsers"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthlyActiveCopilotCodeReviewUsers"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthlyPassiveCopilotCodeReviewUsers"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chatPanelAgentMode" gorm:"comment:Chat panel agent mode interactions"` + ChatPanelAskMode int `json:"chatPanelAskMode" gorm:"comment:Chat panel ask mode interactions"` + ChatPanelCustomMode int `json:"chatPanelCustomMode" gorm:"comment:Chat panel custom mode interactions"` + ChatPanelEditMode int `json:"chatPanelEditMode" gorm:"comment:Chat panel edit mode interactions"` + ChatPanelPlanMode int `json:"chatPanelPlanMode" gorm:"comment:Chat panel plan mode interactions"` + ChatPanelUnknownMode int `json:"chatPanelUnknownMode" gorm:"comment:Chat panel unknown mode interactions"` + + // Pull request metrics (expanded) + PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` + PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` + PRTotalMerged int `json:"prTotalMerged" gorm:"comment:Total PRs merged"` + PRMedianMinutesToMerge float64 `json:"prMedianMinutesToMerge" gorm:"comment:Median minutes to merge PRs"` + PRTotalSuggestions int `json:"prTotalSuggestions" gorm:"comment:Total PR review suggestions"` + PRTotalAppliedSuggestions int `json:"prTotalAppliedSuggestions" gorm:"comment:Total applied PR suggestions"` + PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` + PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + PRTotalMergedCreatedByCopilot int `json:"prTotalMergedCreatedByCopilot" gorm:"comment:Merged PRs created by Copilot"` + PRTotalMergedReviewedByCopilot int `json:"prTotalMergedReviewedByCopilot" gorm:"comment:Merged PRs reviewed by Copilot"` + PRMedianMinToMergeCopilotAuthored float64 `json:"prMedianMinToMergeCopilotAuthored" gorm:"comment:Median min to merge Copilot-authored PRs"` + PRMedianMinToMergeCopilotReviewed float64 `json:"prMedianMinToMergeCopilotReviewed" gorm:"comment:Median min to merge Copilot-reviewed PRs"` + PRTotalCopilotSuggestions int `json:"prTotalCopilotSuggestions" gorm:"comment:Total Copilot review suggestions"` + PRTotalCopilotAppliedSuggestions int `json:"prTotalCopilotAppliedSuggestions" gorm:"comment:Total Copilot applied suggestions"` + + CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotEnterpriseDailyMetrics) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +// GhCopilotMetricsByIde stores enterprise/org metrics broken down by IDE. +type GhCopilotMetricsByIde struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + Ide string `gorm:"primaryKey;type:varchar(50)" json:"ide"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotMetricsByIde) TableName() string { + return "_tool_copilot_metrics_by_ide" +} + +// GhCopilotMetricsByFeature stores enterprise/org metrics broken down by feature. +type GhCopilotMetricsByFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotMetricsByFeature) TableName() string { + return "_tool_copilot_metrics_by_feature" +} + +// GhCopilotMetricsByLanguageFeature stores metrics broken down by language and feature. +type GhCopilotMetricsByLanguageFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + Language string `gorm:"primaryKey;type:varchar(50)" json:"language"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotCodeMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotMetricsByLanguageFeature) TableName() string { + return "_tool_copilot_metrics_by_language_feature" +} + +// GhCopilotMetricsByLanguageModel stores metrics broken down by language and model. +type GhCopilotMetricsByLanguageModel struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + Language string `gorm:"primaryKey;type:varchar(50)" json:"language"` + Model string `gorm:"primaryKey;type:varchar(100)" json:"model"` + + CopilotCodeMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotMetricsByLanguageModel) TableName() string { + return "_tool_copilot_metrics_by_language_model" +} + +// GhCopilotMetricsByModelFeature stores metrics broken down by model and feature. +type GhCopilotMetricsByModelFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + Model string `gorm:"primaryKey;type:varchar(100)" json:"model"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotMetricsByModelFeature) TableName() string { + return "_tool_copilot_metrics_by_model_feature" +} diff --git a/backend/plugins/gh-copilot/models/language_metrics.go b/backend/plugins/gh-copilot/models/language_metrics.go new file mode 100644 index 00000000000..c183abd8309 --- /dev/null +++ b/backend/plugins/gh-copilot/models/language_metrics.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotLanguageMetrics represents engagement statistics broken down by editor and language +// from the organization usage report downloads. +type GhCopilotLanguageMetrics struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + Editor string `gorm:"primaryKey;type:varchar(50)"` + Language string `gorm:"primaryKey;type:varchar(50)"` + + EngagedUsers int `json:"engagedUsers"` + Suggestions int `json:"suggestions"` + Acceptances int `json:"acceptances"` + LinesSuggested int `json:"linesSuggested"` + LinesAccepted int `json:"linesAccepted"` + common.NoPKModel +} + +func (GhCopilotLanguageMetrics) TableName() string { + return "_tool_copilot_org_language_metrics" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20250100_initialize.go b/backend/plugins/gh-copilot/models/migrationscripts/20250100_initialize.go new file mode 100644 index 00000000000..53dee4fbcb2 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20250100_initialize.go @@ -0,0 +1,150 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addCopilotInitialTables creates the initial Copilot tool-layer tables. +type addCopilotInitialTables struct{} + +func (script *addCopilotInitialTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &ghCopilotConnection20250100{}, + &ghCopilotScope20250100{}, + &ghCopilotOrgMetrics20250100{}, + &ghCopilotLanguageMetrics20250100{}, + &ghCopilotSeat20250100{}, + ) +} + +type noPKModel20250100 struct { + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` +} + +type ghCopilotConnection20250100 struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + Endpoint string `gorm:"type:varchar(255)" json:"endpoint"` + Proxy string `gorm:"type:varchar(255)" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `json:"token"` + Organization string `gorm:"type:varchar(255)" json:"organization"` +} + +func (ghCopilotConnection20250100) TableName() string { + return "_tool_copilot_connections" +} + +type ghCopilotScope20250100 struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + Id string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Organization string `json:"organization" gorm:"type:varchar(255)"` + ImplementationDate *time.Time `json:"implementationDate"` + BaselinePeriodDays int `json:"baselinePeriodDays" gorm:"default:90"` + SeatsLastSyncedAt *time.Time `json:"seatsLastSyncedAt"` +} + +func (ghCopilotScope20250100) TableName() string { + return "_tool_copilot_scopes" +} + +type ghCopilotOrgMetrics20250100 struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + + TotalActiveUsers int `json:"totalActiveUsers"` + TotalEngagedUsers int `json:"totalEngagedUsers"` + CompletionSuggestions int `json:"completionSuggestions"` + CompletionAcceptances int `json:"completionAcceptances"` + CompletionLinesSuggested int `json:"completionLinesSuggested"` + CompletionLinesAccepted int `json:"completionLinesAccepted"` + IdeChats int `json:"ideChats"` + IdeChatCopyEvents int `json:"ideChatCopyEvents"` + IdeChatInsertionEvents int `json:"ideChatInsertionEvents"` + IdeChatEngagedUsers int `json:"ideChatEngagedUsers"` + DotcomChats int `json:"dotcomChats"` + DotcomChatEngagedUsers int `json:"dotcomChatEngagedUsers"` + PRSummariesCreated int `json:"prSummariesCreated"` + PREngagedUsers int `json:"prEngagedUsers"` + SeatActiveCount int `json:"seatActiveCount"` + SeatTotal int `json:"seatTotal"` + + archived.NoPKModel +} + +func (ghCopilotOrgMetrics20250100) TableName() string { + return "_tool_copilot_org_daily_metrics" +} + +type ghCopilotLanguageMetrics20250100 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + Editor string `gorm:"primaryKey;type:varchar(50)"` + Language string `gorm:"primaryKey;type:varchar(50)"` + + EngagedUsers int `json:"engagedUsers"` + Suggestions int `json:"suggestions"` + Acceptances int `json:"acceptances"` + LinesSuggested int `json:"linesSuggested"` + LinesAccepted int `json:"linesAccepted"` + + noPKModel20250100 +} + +func (ghCopilotLanguageMetrics20250100) TableName() string { + return "_tool_copilot_org_language_metrics" +} + +type ghCopilotSeat20250100 struct { + ConnectionId uint64 `gorm:"primaryKey"` + Organization string `gorm:"primaryKey;type:varchar(255)"` + UserLogin string `gorm:"primaryKey;type:varchar(255)"` + UserId int64 `gorm:"index"` + PlanType string `gorm:"type:varchar(32)"` + CreatedAt time.Time + LastActivityAt *time.Time + LastActivityEditor string + LastAuthenticatedAt *time.Time + PendingCancellationDate *time.Time + UpdatedAt time.Time +} + +func (ghCopilotSeat20250100) TableName() string { + return "_tool_copilot_seats" +} + +func (*addCopilotInitialTables) Version() uint64 { + return 20250100000000 +} + +func (*addCopilotInitialTables) Name() string { + return "copilot init tables" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260104_add_raw_data_origin_to_language_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260104_add_raw_data_origin_to_language_metrics.go new file mode 100644 index 00000000000..e5e98e15b5d --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260104_add_raw_data_origin_to_language_metrics.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addRawDataOriginToCopilotLanguageMetrics ensures _tool_copilot_language_metrics includes RawDataOrigin columns. +// This is required by StatefulApiExtractor, which attaches provenance to extracted records. +type addRawDataOriginToCopilotLanguageMetrics struct{} + +type ghCopilotLanguageMetrics20260104 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + Editor string `gorm:"primaryKey;type:varchar(50)"` + Language string `gorm:"primaryKey;type:varchar(50)"` + + EngagedUsers int `json:"engagedUsers"` + Suggestions int `json:"suggestions"` + Acceptances int `json:"acceptances"` + LinesSuggested int `json:"linesSuggested"` + LinesAccepted int `json:"linesAccepted"` + + archived.NoPKModel +} + +func (ghCopilotLanguageMetrics20260104) TableName() string { + return "_tool_copilot_org_language_metrics" +} + +func (script *addRawDataOriginToCopilotLanguageMetrics) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &ghCopilotLanguageMetrics20260104{}, + ) +} + +func (*addRawDataOriginToCopilotLanguageMetrics) Version() uint64 { + return 20260104000000 +} + +func (*addRawDataOriginToCopilotLanguageMetrics) Name() string { + return "copilot add raw data origin to language metrics" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260105_add_raw_data_origin_to_seats.go b/backend/plugins/gh-copilot/models/migrationscripts/20260105_add_raw_data_origin_to_seats.go new file mode 100644 index 00000000000..2a921912d6d --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260105_add_raw_data_origin_to_seats.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addRawDataOriginToCopilotSeats ensures _tool_copilot_seats includes RawDataOrigin columns. +// This is required by ApiExtractor/StatefulApiExtractor, which attach provenance to extracted records. +type addRawDataOriginToCopilotSeats struct{} + +type ghCopilotSeat20260105 struct { + ConnectionId uint64 `gorm:"primaryKey"` + Organization string `gorm:"primaryKey;type:varchar(255)"` + UserLogin string `gorm:"primaryKey;type:varchar(255)"` + UserId int64 `gorm:"index"` + PlanType string `gorm:"type:varchar(32)"` + CreatedAt time.Time + LastActivityAt *time.Time + LastActivityEditor string + LastAuthenticatedAt *time.Time + PendingCancellationDate *time.Time + UpdatedAt time.Time + + archived.RawDataOrigin +} + +func (ghCopilotSeat20260105) TableName() string { + return "_tool_copilot_seats" +} + +func (script *addRawDataOriginToCopilotSeats) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &ghCopilotSeat20260105{}, + ) +} + +func (*addRawDataOriginToCopilotSeats) Version() uint64 { + return 20260105000000 +} + +func (*addRawDataOriginToCopilotSeats) Name() string { + return "copilot add raw data origin to seats" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260116_add_name_fields_to_scopes.go b/backend/plugins/gh-copilot/models/migrationscripts/20260116_add_name_fields_to_scopes.go new file mode 100644 index 00000000000..69df2c0b620 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260116_add_name_fields_to_scopes.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +// addNameFieldsToScopes adds name and fullName columns to _tool_copilot_scopes. +// These fields are required by the UI for displaying data scopes in the connection page. +type addNameFieldsToScopes struct{} + +type ghCopilotScope20260116 struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + Id string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Organization string `json:"organization" gorm:"type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" gorm:"type:varchar(255)"` + ImplementationDate *time.Time `json:"implementationDate"` + BaselinePeriodDays int `json:"baselinePeriodDays" gorm:"default:90"` + SeatsLastSyncedAt *time.Time `json:"seatsLastSyncedAt"` +} + +func (ghCopilotScope20260116) TableName() string { + return "_tool_copilot_scopes" +} + +func (script *addNameFieldsToScopes) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &ghCopilotScope20260116{}, + ) +} + +func (*addNameFieldsToScopes) Version() uint64 { + return 20260116000000 +} + +func (*addNameFieldsToScopes) Name() string { + return "copilot add name fields to scopes" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260121_add_scope_configs.go b/backend/plugins/gh-copilot/models/migrationscripts/20260121_add_scope_configs.go new file mode 100644 index 00000000000..4bd77391f1a --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260121_add_scope_configs.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addScopeConfig20260121 struct{} + +type scopeConfig20260121 struct { + archived.Model + Entities []string `gorm:"type:json;serializer:json" json:"entities" mapstructure:"entities"` + ConnectionId uint64 `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex" validate:"required"` + ImplementationDate *time.Time `json:"implementationDate" mapstructure:"implementationDate"` + BaselinePeriodDays int `json:"baselinePeriodDays" mapstructure:"baselinePeriodDays" gorm:"default:90"` +} + +func (scopeConfig20260121) TableName() string { + return "_tool_copilot_scope_configs" +} + +func (*addScopeConfig20260121) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &scopeConfig20260121{}) +} + +func (*addScopeConfig20260121) Version() uint64 { + return 20260121000000 +} + +func (*addScopeConfig20260121) Name() string { + return "Add scope_configs table for GitHub Copilot impact analysis" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260212_add_pr_fields_to_enterprise_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260212_add_pr_fields_to_enterprise_metrics.go new file mode 100644 index 00000000000..6da416668a1 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260212_add_pr_fields_to_enterprise_metrics.go @@ -0,0 +1,349 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addPRFieldsToEnterpriseMetrics struct{} + +// Properly exported embedded struct for activity metrics +type ActivityMetrics20260212v2 struct { + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int +} + +type CodeMetrics20260212v2 struct { + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int +} + +// Enterprise daily metrics with all fields including PR stats +type enterpriseDailyMetrics20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + EnterpriseId string `gorm:"type:varchar(100)"` + DailyActiveUsers int + WeeklyActiveUsers int + MonthlyActiveUsers int + MonthlyActiveChatUsers int + MonthlyActiveAgentUsers int + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + PRTotalReviewed int + PRTotalCreated int + PRTotalCreatedByCopilot int + PRTotalReviewedByCopilot int + archived.NoPKModel +} + +func (enterpriseDailyMetrics20260212v2) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +type metricsByIde20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Ide string `gorm:"primaryKey;type:varchar(50)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (metricsByIde20260212v2) TableName() string { + return "_tool_copilot_metrics_by_ide" +} + +type metricsByFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (metricsByFeature20260212v2) TableName() string { + return "_tool_copilot_metrics_by_feature" +} + +type metricsByLanguageFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (metricsByLanguageFeature20260212v2) TableName() string { + return "_tool_copilot_metrics_by_language_feature" +} + +type metricsByLanguageModel20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Model string `gorm:"primaryKey;type:varchar(100)"` + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (metricsByLanguageModel20260212v2) TableName() string { + return "_tool_copilot_metrics_by_language_model" +} + +type metricsByModelFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Model string `gorm:"primaryKey;type:varchar(100)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (metricsByModelFeature20260212v2) TableName() string { + return "_tool_copilot_metrics_by_model_feature" +} + +// User metrics tables +type userDailyMetrics20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + EnterpriseId string `gorm:"type:varchar(100)"` + UserLogin string `gorm:"type:varchar(255);index"` + UsedAgent bool + UsedChat bool + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userDailyMetrics20260212v2) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +type userMetricsByIde20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Ide string `gorm:"primaryKey;type:varchar(50)"` + LastKnownPluginName string `gorm:"type:varchar(100)"` + LastKnownPluginVersion string `gorm:"type:varchar(50)"` + LastKnownIdeVersion string `gorm:"type:varchar(50)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userMetricsByIde20260212v2) TableName() string { + return "_tool_copilot_user_metrics_by_ide" +} + +type userMetricsByFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userMetricsByFeature20260212v2) TableName() string { + return "_tool_copilot_user_metrics_by_feature" +} + +type userMetricsByLanguageFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userMetricsByLanguageFeature20260212v2) TableName() string { + return "_tool_copilot_user_metrics_by_language_feature" +} + +type userMetricsByLanguageModel20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Model string `gorm:"primaryKey;type:varchar(100)"` + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userMetricsByLanguageModel20260212v2) TableName() string { + return "_tool_copilot_user_metrics_by_language_model" +} + +type userMetricsByModelFeature20260212v2 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Model string `gorm:"primaryKey;type:varchar(100)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + UserInitiatedInteractionCount int + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int + archived.NoPKModel +} + +func (userMetricsByModelFeature20260212v2) TableName() string { + return "_tool_copilot_user_metrics_by_model_feature" +} + +func (script *addPRFieldsToEnterpriseMetrics) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + // Drop all metric tables created by v2 migration (they have missing columns + // due to unexported embedded struct bug in GORM). Data will be re-collected. + if err := db.DropTables( + "_tool_copilot_enterprise_daily_metrics", + "_tool_copilot_metrics_by_ide", + "_tool_copilot_metrics_by_feature", + "_tool_copilot_metrics_by_language_feature", + "_tool_copilot_metrics_by_language_model", + "_tool_copilot_metrics_by_model_feature", + "_tool_copilot_user_daily_metrics", + "_tool_copilot_user_metrics_by_ide", + "_tool_copilot_user_metrics_by_feature", + "_tool_copilot_user_metrics_by_language_feature", + "_tool_copilot_user_metrics_by_language_model", + "_tool_copilot_user_metrics_by_model_feature", + ); err != nil { + basicRes.GetLogger().Warn(err, "Failed to drop tables for recreation") + } + + // Recreate with all columns properly defined (no embedded structs) + return migrationhelper.AutoMigrateTables(basicRes, + &enterpriseDailyMetrics20260212v2{}, + &metricsByIde20260212v2{}, + &metricsByFeature20260212v2{}, + &metricsByLanguageFeature20260212v2{}, + &metricsByLanguageModel20260212v2{}, + &metricsByModelFeature20260212v2{}, + &userDailyMetrics20260212v2{}, + &userMetricsByIde20260212v2{}, + &userMetricsByFeature20260212v2{}, + &userMetricsByLanguageFeature20260212v2{}, + &userMetricsByLanguageModel20260212v2{}, + &userMetricsByModelFeature20260212v2{}, + ) +} + +func (*addPRFieldsToEnterpriseMetrics) Version() uint64 { + return 20260212100000 +} + +func (*addPRFieldsToEnterpriseMetrics) Name() string { + return "Recreate metric tables with all columns and PR fields" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260212_v2_usage_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260212_v2_usage_metrics.go new file mode 100644 index 00000000000..7ede2b680d8 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260212_v2_usage_metrics.go @@ -0,0 +1,349 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type migrateToUsageMetricsV2 struct{} + +// --- Snapshot structs for migration (avoid importing models package to prevent drift) --- + +// Connection: add Enterprise column +type connection20260212 struct { + Enterprise string `gorm:"type:varchar(100)"` +} + +func (connection20260212) TableName() string { + return "_tool_copilot_connections" +} + +// Scope: add Enterprise column +type scope20260212 struct { + Enterprise string `gorm:"type:varchar(100)"` +} + +func (scope20260212) TableName() string { + return "_tool_copilot_scopes" +} + +// --- Common embedded structs --- + +type activityMetrics20260212 struct { + UserInitiatedInteractionCount int `gorm:"comment:Chat messages and inline prompts initiated by user"` + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int +} + +type codeMetrics20260212 struct { + CodeGenerationActivityCount int + CodeAcceptanceActivityCount int + LocSuggestedToAddSum int + LocSuggestedToDeleteSum int + LocAddedSum int + LocDeletedSum int +} + +// --- Enterprise metrics tables --- + +type enterpriseDailyMetrics20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + EnterpriseId string `gorm:"type:varchar(100)"` + DailyActiveUsers int + WeeklyActiveUsers int + MonthlyActiveUsers int + MonthlyActiveChatUsers int + MonthlyActiveAgentUsers int + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (enterpriseDailyMetrics20260212) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +type metricsByIde20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Ide string `gorm:"primaryKey;type:varchar(50)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (metricsByIde20260212) TableName() string { + return "_tool_copilot_metrics_by_ide" +} + +type metricsByFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (metricsByFeature20260212) TableName() string { + return "_tool_copilot_metrics_by_feature" +} + +type metricsByLanguageFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + codeMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (metricsByLanguageFeature20260212) TableName() string { + return "_tool_copilot_metrics_by_language_feature" +} + +type metricsByLanguageModel20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Model string `gorm:"primaryKey;type:varchar(100)"` + codeMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (metricsByLanguageModel20260212) TableName() string { + return "_tool_copilot_metrics_by_language_model" +} + +type metricsByModelFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + Model string `gorm:"primaryKey;type:varchar(100)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (metricsByModelFeature20260212) TableName() string { + return "_tool_copilot_metrics_by_model_feature" +} + +// --- User metrics tables --- + +type userDailyMetrics20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + EnterpriseId string `gorm:"type:varchar(100)"` + UserLogin string `gorm:"type:varchar(255);index"` + UsedAgent bool + UsedChat bool + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userDailyMetrics20260212) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +type userMetricsByIde20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Ide string `gorm:"primaryKey;type:varchar(50)"` + LastKnownPluginName string `gorm:"type:varchar(100)"` + LastKnownPluginVersion string `gorm:"type:varchar(50)"` + LastKnownIdeVersion string `gorm:"type:varchar(50)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userMetricsByIde20260212) TableName() string { + return "_tool_copilot_user_metrics_by_ide" +} + +type userMetricsByFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userMetricsByFeature20260212) TableName() string { + return "_tool_copilot_user_metrics_by_feature" +} + +type userMetricsByLanguageFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + codeMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userMetricsByLanguageFeature20260212) TableName() string { + return "_tool_copilot_user_metrics_by_language_feature" +} + +type userMetricsByLanguageModel20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Language string `gorm:"primaryKey;type:varchar(50)"` + Model string `gorm:"primaryKey;type:varchar(100)"` + codeMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userMetricsByLanguageModel20260212) TableName() string { + return "_tool_copilot_user_metrics_by_language_model" +} + +type userMetricsByModelFeature20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + Model string `gorm:"primaryKey;type:varchar(100)"` + Feature string `gorm:"primaryKey;type:varchar(100)"` + activityMetrics20260212 `gorm:"embedded"` + archived.NoPKModel +} + +func (userMetricsByModelFeature20260212) TableName() string { + return "_tool_copilot_user_metrics_by_model_feature" +} + +// --- New org metrics tables (replacing old ones) --- + +type orgDailyMetrics20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + TotalActiveUsers int + TotalEngagedUsers int + CompletionSuggestions int + CompletionAcceptances int + CompletionLinesSuggested int + CompletionLinesAccepted int + IdeChats int + IdeChatCopyEvents int + IdeChatInsertionEvents int + IdeChatEngagedUsers int + DotcomChats int + DotcomChatEngagedUsers int + PRSummariesCreated int + PREngagedUsers int + SeatActiveCount int + SeatTotal int + archived.NoPKModel +} + +func (orgDailyMetrics20260212) TableName() string { + return "_tool_copilot_org_daily_metrics" +} + +type orgLanguageMetrics20260212 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Date time.Time `gorm:"primaryKey;type:date"` + Editor string `gorm:"primaryKey;type:varchar(50)"` + Language string `gorm:"primaryKey;type:varchar(50)"` + EngagedUsers int + Suggestions int + Acceptances int + LinesSuggested int + LinesAccepted int + archived.NoPKModel +} + +func (orgLanguageMetrics20260212) TableName() string { + return "_tool_copilot_org_language_metrics" +} + +func (script *migrateToUsageMetricsV2) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + // Drop legacy metrics tables (data must be re-collected from new API) + if err := db.DropTables( + "_tool_copilot_org_metrics", + "_tool_copilot_language_metrics", + ); err != nil { + basicRes.GetLogger().Warn(err, "Failed to drop legacy copilot tables (may not exist)") + } + + // Add Enterprise column to connections and scopes + if err := migrationhelper.AutoMigrateTables(basicRes, + &connection20260212{}, + &scope20260212{}, + ); err != nil { + return err + } + + // Create all new tables + return migrationhelper.AutoMigrateTables(basicRes, + // New org metrics (replacing dropped tables) + &orgDailyMetrics20260212{}, + &orgLanguageMetrics20260212{}, + // Enterprise metrics + &enterpriseDailyMetrics20260212{}, + &metricsByIde20260212{}, + &metricsByFeature20260212{}, + &metricsByLanguageFeature20260212{}, + &metricsByLanguageModel20260212{}, + &metricsByModelFeature20260212{}, + // User metrics + &userDailyMetrics20260212{}, + &userMetricsByIde20260212{}, + &userMetricsByFeature20260212{}, + &userMetricsByLanguageFeature20260212{}, + &userMetricsByLanguageModel20260212{}, + &userMetricsByModelFeature20260212{}, + ) +} + +func (*migrateToUsageMetricsV2) Version() uint64 { + return 20260212000000 +} + +func (*migrateToUsageMetricsV2) Name() string { + return "Migrate GitHub Copilot to Usage Metrics Report API v2" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260303_add_organization_id_to_user_metrics.go b/backend/plugins/gh-copilot/models/migrationscripts/20260303_add_organization_id_to_user_metrics.go new file mode 100644 index 00000000000..d868be69084 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260303_add_organization_id_to_user_metrics.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addOrganizationIdToUserMetrics)(nil) + +type addOrganizationIdToUserMetrics struct{} + +func (script *addOrganizationIdToUserMetrics) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + return db.AutoMigrate(&userDailyMetrics20260303{}) +} + +func (*addOrganizationIdToUserMetrics) Version() uint64 { + return 20260303000000 +} + +func (*addOrganizationIdToUserMetrics) Name() string { + return "add organization_id to user daily metrics" +} + +type userDailyMetrics20260303 struct { + OrganizationId string `gorm:"type:varchar(100)"` +} + +func (userDailyMetrics20260303) TableName() string { + return "_tool_copilot_user_daily_metrics" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go new file mode 100644 index 00000000000..f676b0e4bee --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCopilotMetricsGaps struct{} + +// --- Enterprise daily metrics: new columns --- + +type enterpriseDailyMetrics20260527 struct { + // CLI + DailyActiveCliUsers int + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int + DailyPassiveCopilotCodeReviewUsers int + WeeklyActiveCopilotCodeReviewUsers int + WeeklyPassiveCopilotCodeReviewUsers int + MonthlyActiveCopilotCodeReviewUsers int + MonthlyPassiveCopilotCodeReviewUsers int + + // Chat panel mode breakdown + ChatPanelAgentMode int + ChatPanelAskMode int + ChatPanelCustomMode int + ChatPanelEditMode int + ChatPanelPlanMode int + ChatPanelUnknownMode int + + // Expanded PR metrics + PRTotalMerged int + PRMedianMinutesToMerge float64 + PRTotalSuggestions int + PRTotalAppliedSuggestions int + PRTotalMergedCreatedByCopilot int + PRTotalMergedReviewedByCopilot int + PRMedianMinToMergeCopilotAuthored float64 + PRMedianMinToMergeCopilotReviewed float64 + PRTotalCopilotSuggestions int + PRTotalCopilotAppliedSuggestions int + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (enterpriseDailyMetrics20260527) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +// --- User daily metrics: new columns --- + +type userDailyMetrics20260527 struct { + UsedCli bool + UsedCopilotCodeReviewActive bool + UsedCopilotCodeReviewPassive bool + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (userDailyMetrics20260527) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +// --- Seat: new columns --- + +type seat20260527 struct { + UserName string `gorm:"type:varchar(255)"` + UserEmail string `gorm:"type:varchar(255)"` + AssigningTeamId int64 + AssigningTeamName string `gorm:"type:varchar(255)"` + AssigningTeamSlug string `gorm:"type:varchar(255)"` +} + +func (seat20260527) TableName() string { + return "_tool_copilot_seats" +} + +// --- User-teams: new table --- + +type userTeam20260527 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + + UserLogin string `gorm:"type:varchar(255);index"` + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + TeamSlug string `gorm:"type:varchar(255)"` + + archived.NoPKModel +} + +func (userTeam20260527) TableName() string { + return "_tool_copilot_user_teams" +} + +func (script *addCopilotMetricsGaps) Up(basicRes context.BasicRes) errors.Error { + // Add new columns to existing tables + if err := migrationhelper.AutoMigrateTables(basicRes, + &enterpriseDailyMetrics20260527{}, + &userDailyMetrics20260527{}, + &seat20260527{}, + ); err != nil { + return err + } + + // Create new user-teams table + return migrationhelper.AutoMigrateTables(basicRes, + &userTeam20260527{}, + ) +} + +func (*addCopilotMetricsGaps) Version() uint64 { + return 20260527000000 +} + +func (*addCopilotMetricsGaps) Name() string { + return "Add Copilot metrics gaps: CLI, code review, chat modes, PR expansion, user-teams" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go new file mode 100644 index 00000000000..399735695e0 --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import "github.com/apache/incubator-devlake/core/plugin" + +// All returns the ordered list of migration scripts for the Copilot plugin. +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addCopilotInitialTables), + new(addRawDataOriginToCopilotSeats), + new(addRawDataOriginToCopilotLanguageMetrics), + new(addNameFieldsToScopes), + new(addScopeConfig20260121), + new(migrateToUsageMetricsV2), + new(addPRFieldsToEnterpriseMetrics), + new(addOrganizationIdToUserMetrics), + new(addCopilotMetricsGaps), + } +} diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go new file mode 100644 index 00000000000..5143ce5f8b7 --- /dev/null +++ b/backend/plugins/gh-copilot/models/models.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/dal" + +// GetTablesInfo returns the list of tool-layer tables managed by the Copilot plugin. +func GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + // Connection, scope, config + &GhCopilotConnection{}, + &GhCopilotScope{}, + &GhCopilotScopeConfig{}, + // Org-level metrics (from organization reports) + &GhCopilotOrgMetrics{}, + &GhCopilotLanguageMetrics{}, + // Enterprise-level metrics (from enterprise reports) + &GhCopilotEnterpriseDailyMetrics{}, + &GhCopilotMetricsByIde{}, + &GhCopilotMetricsByFeature{}, + &GhCopilotMetricsByLanguageFeature{}, + &GhCopilotMetricsByLanguageModel{}, + &GhCopilotMetricsByModelFeature{}, + // User-level metrics (from enterprise user reports) + &GhCopilotUserDailyMetrics{}, + &GhCopilotUserMetricsByIde{}, + &GhCopilotUserMetricsByFeature{}, + &GhCopilotUserMetricsByLanguageFeature{}, + &GhCopilotUserMetricsByLanguageModel{}, + &GhCopilotUserMetricsByModelFeature{}, + // Seat assignments + &GhCopilotSeat{}, + // User-team mappings + &GhCopilotUserTeam{}, + } +} diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go new file mode 100644 index 00000000000..ef5b3eff6f5 --- /dev/null +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "testing" + +func TestGetTablesInfo(t *testing.T) { + tables := GetTablesInfo() + expected := map[string]bool{ + (&GhCopilotConnection{}).TableName(): false, + (&GhCopilotScope{}).TableName(): false, + (&GhCopilotScopeConfig{}).TableName(): false, + (&GhCopilotOrgMetrics{}).TableName(): false, + (&GhCopilotLanguageMetrics{}).TableName(): false, + (&GhCopilotEnterpriseDailyMetrics{}).TableName(): false, + (&GhCopilotMetricsByIde{}).TableName(): false, + (&GhCopilotMetricsByFeature{}).TableName(): false, + (&GhCopilotMetricsByLanguageFeature{}).TableName(): false, + (&GhCopilotMetricsByLanguageModel{}).TableName(): false, + (&GhCopilotMetricsByModelFeature{}).TableName(): false, + (&GhCopilotUserDailyMetrics{}).TableName(): false, + (&GhCopilotUserMetricsByIde{}).TableName(): false, + (&GhCopilotUserMetricsByFeature{}).TableName(): false, + (&GhCopilotUserMetricsByLanguageFeature{}).TableName(): false, + (&GhCopilotUserMetricsByLanguageModel{}).TableName(): false, + (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, + (&GhCopilotSeat{}).TableName(): false, + (&GhCopilotUserTeam{}).TableName(): false, + } + + if len(tables) != len(expected) { + t.Fatalf("unexpected number of tables: want %d, got %d", len(expected), len(tables)) + } + + for _, table := range tables { + tableName := table.TableName() + if _, ok := expected[tableName]; !ok { + t.Fatalf("unexpected table registered: %s", tableName) + } + expected[tableName] = true + } + + for name, seen := range expected { + if !seen { + t.Fatalf("table not registered: %s", name) + } + } +} diff --git a/backend/plugins/gh-copilot/models/org_metrics.go b/backend/plugins/gh-copilot/models/org_metrics.go new file mode 100644 index 00000000000..22401c80e35 --- /dev/null +++ b/backend/plugins/gh-copilot/models/org_metrics.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotOrgMetrics captures daily organization-level Copilot adoption metrics +// from the organization usage report downloads. +type GhCopilotOrgMetrics struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + + TotalActiveUsers int `json:"totalActiveUsers"` + TotalEngagedUsers int `json:"totalEngagedUsers"` + CompletionSuggestions int `json:"completionSuggestions"` + CompletionAcceptances int `json:"completionAcceptances"` + CompletionLinesSuggested int `json:"completionLinesSuggested"` + CompletionLinesAccepted int `json:"completionLinesAccepted"` + IdeChats int `json:"ideChats"` + IdeChatCopyEvents int `json:"ideChatCopyEvents"` + IdeChatInsertionEvents int `json:"ideChatInsertionEvents"` + IdeChatEngagedUsers int `json:"ideChatEngagedUsers"` + DotcomChats int `json:"dotcomChats"` + DotcomChatEngagedUsers int `json:"dotcomChatEngagedUsers"` + PRSummariesCreated int `json:"prSummariesCreated"` + PREngagedUsers int `json:"prEngagedUsers"` + SeatActiveCount int `json:"seatActiveCount"` + SeatTotal int `json:"seatTotal"` + + common.NoPKModel +} + +func (GhCopilotOrgMetrics) TableName() string { + return "_tool_copilot_org_daily_metrics" +} diff --git a/backend/plugins/gh-copilot/models/scope.go b/backend/plugins/gh-copilot/models/scope.go new file mode 100644 index 00000000000..426f30169a2 --- /dev/null +++ b/backend/plugins/gh-copilot/models/scope.go @@ -0,0 +1,96 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "strings" + "time" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +// GhCopilotScope represents an organization or enterprise-level collection scope. +type GhCopilotScope struct { + common.Scope `mapstructure:",squash"` + Id string `json:"id" mapstructure:"id" gorm:"primaryKey;type:varchar(255)"` + Organization string `json:"organization" mapstructure:"organization" gorm:"type:varchar(255)"` + Enterprise string `json:"enterprise,omitempty" mapstructure:"enterprise" gorm:"type:varchar(100)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + FullName string `json:"fullName" mapstructure:"fullName" gorm:"type:varchar(255)"` + ImplementationDate *time.Time `json:"implementationDate" mapstructure:"implementationDate"` + BaselinePeriodDays int `json:"baselinePeriodDays" mapstructure:"baselinePeriodDays"` + SeatsLastSyncedAt *time.Time `json:"seatsLastSyncedAt" mapstructure:"seatsLastSyncedAt"` +} + +// IsEnterprise returns true if this scope targets an enterprise. +func (s *GhCopilotScope) IsEnterprise() bool { + return s != nil && strings.TrimSpace(s.Enterprise) != "" +} + +func (GhCopilotScope) TableName() string { + return "_tool_copilot_scopes" +} + +func (s *GhCopilotScope) BeforeSave(tx *gorm.DB) error { + // Populate Name and FullName from Organization and Id + if s.Name == "" { + s.Name = s.ScopeName() + } + if s.FullName == "" { + s.FullName = s.ScopeFullName() + } + // Validate and normalize BaselinePeriodDays (7-365 range, default 90) + if s.BaselinePeriodDays < 7 { + s.BaselinePeriodDays = 90 // Default to 90 days + } else if s.BaselinePeriodDays > 365 { + s.BaselinePeriodDays = 365 // Cap at 1 year + } + return nil +} + +func (s GhCopilotScope) ScopeId() string { + return s.Id +} + +func (s GhCopilotScope) ScopeName() string { + if s.Id != "" { + return s.Id + } + return s.Organization +} + +func (s GhCopilotScope) ScopeFullName() string { + return s.ScopeName() +} + +func (s GhCopilotScope) ScopeParams() interface{} { + return &GhCopilotScopeParams{ + ConnectionId: s.ConnectionId, + ScopeId: s.Id, + } +} + +// GhCopilotScopeParams is returned for blueprint configuration. +type GhCopilotScopeParams struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` +} + +var _ plugin.ToolLayerScope = (*GhCopilotScope)(nil) diff --git a/backend/plugins/gh-copilot/models/scope_config.go b/backend/plugins/gh-copilot/models/scope_config.go new file mode 100644 index 00000000000..2618b87e5c1 --- /dev/null +++ b/backend/plugins/gh-copilot/models/scope_config.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +var _ plugin.ToolLayerScopeConfig = (*GhCopilotScopeConfig)(nil) + +// GhCopilotScopeConfig contains configuration for GitHub Copilot data scope. +// This includes settings for the Impact Dashboard analysis. +type GhCopilotScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + // ImplementationDate is the optional rollout milestone date for before/after analysis + ImplementationDate *time.Time `json:"implementationDate" mapstructure:"implementationDate" gorm:"type:datetime"` + // BaselinePeriodDays is the number of days to use for baseline comparison (default: 90) + BaselinePeriodDays int `json:"baselinePeriodDays" mapstructure:"baselinePeriodDays" gorm:"default:90"` +} + +func (GhCopilotScopeConfig) TableName() string { + return "_tool_copilot_scope_configs" +} + +// GetConnectionId implements plugin.ToolLayerScopeConfig. +func (sc GhCopilotScopeConfig) GetConnectionId() uint64 { + return sc.ConnectionId +} + +// BeforeSave validates and normalizes the scope config before saving. +func (sc *GhCopilotScopeConfig) BeforeSave(tx *gorm.DB) error { + // Validate and normalize BaselinePeriodDays (7-365 range, default 90) + if sc.BaselinePeriodDays < 7 { + sc.BaselinePeriodDays = 90 // Default to 90 days + } else if sc.BaselinePeriodDays > 365 { + sc.BaselinePeriodDays = 365 // Cap at 1 year + } + return nil +} diff --git a/backend/plugins/gh-copilot/models/scope_config_test.go b/backend/plugins/gh-copilot/models/scope_config_test.go new file mode 100644 index 00000000000..54091aaa7f1 --- /dev/null +++ b/backend/plugins/gh-copilot/models/scope_config_test.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + "time" +) + +func TestGhCopilotScopeConfig_TableName(t *testing.T) { + sc := GhCopilotScopeConfig{} + expected := "_tool_copilot_scope_configs" + if sc.TableName() != expected { + t.Errorf("TableName() = %v, want %v", sc.TableName(), expected) + } +} + +func TestGhCopilotScopeConfig_BeforeSave_BaselinePeriodDays(t *testing.T) { + tests := []struct { + name string + input int + expected int + }{ + {"zero defaults to 90", 0, 90}, + {"negative defaults to 90", -10, 90}, + {"below minimum (5) defaults to 90", 5, 90}, + {"below minimum (6) defaults to 90", 6, 90}, + {"minimum valid (7) unchanged", 7, 7}, + {"typical value (30) unchanged", 30, 30}, + {"default value (90) unchanged", 90, 90}, + {"high value (180) unchanged", 180, 180}, + {"maximum valid (365) unchanged", 365, 365}, + {"above maximum (400) capped to 365", 400, 365}, + {"far above maximum (1000) capped to 365", 1000, 365}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sc := &GhCopilotScopeConfig{ + BaselinePeriodDays: tt.input, + } + err := sc.BeforeSave(nil) + if err != nil { + t.Errorf("BeforeSave() error = %v, want nil", err) + } + if sc.BaselinePeriodDays != tt.expected { + t.Errorf("BaselinePeriodDays = %v, want %v", sc.BaselinePeriodDays, tt.expected) + } + }) + } +} + +func TestGhCopilotScopeConfig_BeforeSave_PreservesOtherFields(t *testing.T) { + now := time.Now() + sc := &GhCopilotScopeConfig{ + ImplementationDate: &now, + BaselinePeriodDays: 60, + } + + err := sc.BeforeSave(nil) + if err != nil { + t.Errorf("BeforeSave() error = %v, want nil", err) + } + + // ImplementationDate should be preserved + if sc.ImplementationDate == nil || !sc.ImplementationDate.Equal(now) { + t.Error("ImplementationDate was modified unexpectedly") + } + + // BaselinePeriodDays should be unchanged (valid value) + if sc.BaselinePeriodDays != 60 { + t.Errorf("BaselinePeriodDays = %v, want 60", sc.BaselinePeriodDays) + } +} + +func TestGhCopilotScopeConfig_GetConnectionId(t *testing.T) { + sc := GhCopilotScopeConfig{} + sc.ConnectionId = 42 + + if sc.GetConnectionId() != 42 { + t.Errorf("GetConnectionId() = %v, want 42", sc.GetConnectionId()) + } +} diff --git a/backend/plugins/gh-copilot/models/scope_test.go b/backend/plugins/gh-copilot/models/scope_test.go new file mode 100644 index 00000000000..6a3d96a0ec4 --- /dev/null +++ b/backend/plugins/gh-copilot/models/scope_test.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "testing" + +func TestGhCopilotScope_TableName(t *testing.T) { + s := GhCopilotScope{} + expected := "_tool_copilot_scopes" + if s.TableName() != expected { + t.Errorf("TableName() = %v, want %v", s.TableName(), expected) + } +} + +func TestGhCopilotScope_ScopeName(t *testing.T) { + tests := []struct { + name string + id string + organization string + expected string + }{ + {"returns Id when set", "my-org", "fallback-org", "my-org"}, + {"returns Organization when Id empty", "", "my-org", "my-org"}, + {"returns empty when both empty", "", "", ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := GhCopilotScope{ + Id: tt.id, + Organization: tt.organization, + } + if got := s.ScopeName(); got != tt.expected { + t.Errorf("ScopeName() = %v, want %v", got, tt.expected) + } + }) + } +} + +func TestGhCopilotScope_ScopeFullName(t *testing.T) { + s := GhCopilotScope{Id: "test-org"} + if got := s.ScopeFullName(); got != "test-org" { + t.Errorf("ScopeFullName() = %v, want test-org", got) + } +} + +func TestGhCopilotScope_BeforeSave_PopulatesName(t *testing.T) { + tests := []struct { + name string + id string + organization string + initialName string + expectedName string + expectedFullName string + }{ + { + name: "populates Name from Id when empty", + id: "my-org", + organization: "fallback", + initialName: "", + expectedName: "my-org", + expectedFullName: "my-org", + }, + { + name: "populates Name from Organization when Id empty", + id: "", + organization: "my-org", + initialName: "", + expectedName: "my-org", + expectedFullName: "my-org", + }, + { + name: "preserves existing Name", + id: "org-id", + organization: "org", + initialName: "custom-name", + expectedName: "custom-name", + expectedFullName: "org-id", // FullName still populated from ScopeName + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &GhCopilotScope{ + Id: tt.id, + Organization: tt.organization, + Name: tt.initialName, + } + err := s.BeforeSave(nil) + if err != nil { + t.Errorf("BeforeSave() error = %v, want nil", err) + } + if s.Name != tt.expectedName { + t.Errorf("Name = %v, want %v", s.Name, tt.expectedName) + } + if s.FullName != tt.expectedFullName { + t.Errorf("FullName = %v, want %v", s.FullName, tt.expectedFullName) + } + }) + } +} + +func TestGhCopilotScope_BeforeSave_BaselinePeriodDays(t *testing.T) { + tests := []struct { + name string + input int + expected int + }{ + {"zero defaults to 90", 0, 90}, + {"negative defaults to 90", -10, 90}, + {"below minimum (6) defaults to 90", 6, 90}, + {"minimum valid (7) unchanged", 7, 7}, + {"typical value (90) unchanged", 90, 90}, + {"maximum valid (365) unchanged", 365, 365}, + {"above maximum (400) capped to 365", 400, 365}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &GhCopilotScope{ + Id: "test-org", + BaselinePeriodDays: tt.input, + } + err := s.BeforeSave(nil) + if err != nil { + t.Errorf("BeforeSave() error = %v, want nil", err) + } + if s.BaselinePeriodDays != tt.expected { + t.Errorf("BaselinePeriodDays = %v, want %v", s.BaselinePeriodDays, tt.expected) + } + }) + } +} + +func TestGhCopilotScope_ScopeId(t *testing.T) { + s := GhCopilotScope{Id: "test-id"} + if got := s.ScopeId(); got != "test-id" { + t.Errorf("ScopeId() = %v, want test-id", got) + } +} + +func TestGhCopilotScope_ScopeParams(t *testing.T) { + s := GhCopilotScope{Id: "test-org"} + s.ConnectionId = 42 + + params := s.ScopeParams().(*GhCopilotScopeParams) + if params.ConnectionId != 42 { + t.Errorf("ScopeParams().ConnectionId = %v, want 42", params.ConnectionId) + } + if params.ScopeId != "test-org" { + t.Errorf("ScopeParams().ScopeId = %v, want test-org", params.ScopeId) + } +} diff --git a/backend/plugins/gh-copilot/models/seat.go b/backend/plugins/gh-copilot/models/seat.go new file mode 100644 index 00000000000..d65c80f2e30 --- /dev/null +++ b/backend/plugins/gh-copilot/models/seat.go @@ -0,0 +1,50 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotSeat represents a seat assignment snapshot for Copilot. +type GhCopilotSeat struct { + ConnectionId uint64 `gorm:"primaryKey"` + Organization string `gorm:"primaryKey;type:varchar(255)"` + UserLogin string `gorm:"primaryKey;type:varchar(255)"` + UserId int64 `gorm:"index"` + UserName string `gorm:"type:varchar(255)" json:"userName"` + UserEmail string `gorm:"type:varchar(255)" json:"userEmail"` + PlanType string `gorm:"type:varchar(32)"` + AssigningTeamId int64 `json:"assigningTeamId" gorm:"comment:Team that assigned the seat"` + AssigningTeamName string `json:"assigningTeamName" gorm:"type:varchar(255)"` + AssigningTeamSlug string `json:"assigningTeamSlug" gorm:"type:varchar(255)"` + CreatedAt time.Time + LastActivityAt *time.Time + LastActivityEditor string + LastAuthenticatedAt *time.Time + PendingCancellationDate *time.Time + UpdatedAt time.Time + + common.RawDataOrigin +} + +func (GhCopilotSeat) TableName() string { + return "_tool_copilot_seats" +} diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go new file mode 100644 index 00000000000..18e9134c226 --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -0,0 +1,136 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserDailyMetrics captures per-user daily Copilot usage metrics from enterprise reports. +type GhCopilotUserDailyMetrics struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + UsedAgent bool `json:"usedAgent"` + UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` + + CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserDailyMetrics) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +// GhCopilotUserMetricsByIde stores per-user metrics broken down by IDE. +type GhCopilotUserMetricsByIde struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + Ide string `gorm:"primaryKey;type:varchar(50)" json:"ide"` + + LastKnownPluginName string `json:"lastKnownPluginName" gorm:"type:varchar(100)"` + LastKnownPluginVersion string `json:"lastKnownPluginVersion" gorm:"type:varchar(50)"` + LastKnownIdeVersion string `json:"lastKnownIdeVersion" gorm:"type:varchar(50)"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserMetricsByIde) TableName() string { + return "_tool_copilot_user_metrics_by_ide" +} + +// GhCopilotUserMetricsByFeature stores per-user metrics broken down by feature. +type GhCopilotUserMetricsByFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserMetricsByFeature) TableName() string { + return "_tool_copilot_user_metrics_by_feature" +} + +// GhCopilotUserMetricsByLanguageFeature stores per-user metrics by language and feature. +type GhCopilotUserMetricsByLanguageFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + Language string `gorm:"primaryKey;type:varchar(50)" json:"language"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotCodeMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserMetricsByLanguageFeature) TableName() string { + return "_tool_copilot_user_metrics_by_language_feature" +} + +// GhCopilotUserMetricsByLanguageModel stores per-user metrics by language and model. +type GhCopilotUserMetricsByLanguageModel struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + Language string `gorm:"primaryKey;type:varchar(50)" json:"language"` + Model string `gorm:"primaryKey;type:varchar(100)" json:"model"` + + CopilotCodeMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserMetricsByLanguageModel) TableName() string { + return "_tool_copilot_user_metrics_by_language_model" +} + +// GhCopilotUserMetricsByModelFeature stores per-user metrics by model and feature. +type GhCopilotUserMetricsByModelFeature struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + Model string `gorm:"primaryKey;type:varchar(100)" json:"model"` + Feature string `gorm:"primaryKey;type:varchar(100)" json:"feature"` + + CopilotActivityMetrics `mapstructure:",squash"` + common.NoPKModel +} + +func (GhCopilotUserMetricsByModelFeature) TableName() string { + return "_tool_copilot_user_metrics_by_model_feature" +} diff --git a/backend/plugins/gh-copilot/models/user_team.go b/backend/plugins/gh-copilot/models/user_team.go new file mode 100644 index 00000000000..d04d55ac56f --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_team.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserTeam maps users to teams per day from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +type GhCopilotUserTeam struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + TeamId int64 `gorm:"primaryKey" json:"teamId"` + + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + TeamSlug string `json:"teamSlug" gorm:"type:varchar(255)"` + + common.NoPKModel +} + +func (GhCopilotUserTeam) TableName() string { + return "_tool_copilot_user_teams" +} diff --git a/backend/plugins/gh-copilot/service/api_path.go b/backend/plugins/gh-copilot/service/api_path.go new file mode 100644 index 00000000000..ec593c94671 --- /dev/null +++ b/backend/plugins/gh-copilot/service/api_path.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "fmt" + "net/url" + "strings" +) + +func copilotAPIPath(namespace, slug, resource string) string { + return fmt.Sprintf("%s/%s/%s", namespace, url.PathEscape(strings.TrimSpace(slug)), strings.TrimPrefix(resource, "/")) +} diff --git a/backend/plugins/gh-copilot/service/api_path_test.go b/backend/plugins/gh-copilot/service/api_path_test.go new file mode 100644 index 00000000000..4a57cc1e44b --- /dev/null +++ b/backend/plugins/gh-copilot/service/api_path_test.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCopilotAPIPathPreservesHyphenatedEnterpriseSlug(t *testing.T) { + path := copilotAPIPath("enterprises", "my-enterprise", "copilot/billing/seats") + + assert.Equal(t, "enterprises/my-enterprise/copilot/billing/seats", path) +} diff --git a/backend/plugins/gh-copilot/service/connection_test_helper.go b/backend/plugins/gh-copilot/service/connection_test_helper.go new file mode 100644 index 00000000000..8891476020f --- /dev/null +++ b/backend/plugins/gh-copilot/service/connection_test_helper.go @@ -0,0 +1,221 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + stdctx "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// TestConnectionResult represents the payload returned by the connection test endpoints. +type TestConnectionResult struct { + Success bool `json:"success"` + Message string `json:"message"` + Enterprise string `json:"enterprise,omitempty"` + Organization string `json:"organization,omitempty"` + PlanType string `json:"planType,omitempty"` + TotalSeats int `json:"totalSeats,omitempty"` + ActiveSeats int `json:"activeSeats,omitempty"` +} + +type copilotBillingSummary struct { + Organization string `json:"organization"` + PlanType string `json:"plan_type"` + TotalSeats int `json:"total_seats"` + ActiveSeats int `json:"active_seats"` + ActiveThisCycle int `json:"active_this_cycle"` +} + +// TestConnection exercises the GitHub Copilot billing endpoint to validate credentials. +func TestConnection(ctx stdctx.Context, br corectx.BasicRes, connection *models.GhCopilotConnection) (*TestConnectionResult, errors.Error) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + connection.Normalize() + + hasEnterprise := connection.HasEnterprise() + hasOrg := strings.TrimSpace(connection.Organization) != "" + + if !hasEnterprise && !hasOrg { + return nil, errors.BadInput.New("either enterprise or organization must be specified") + } + + apiClient, err := helper.NewApiClientFromConnection(ctx, br, connection) + if err != nil { + return nil, err + } + apiClient.SetHeaders(map[string]string{ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2026-03-10", + }) + + result := &TestConnectionResult{ + Success: true, + Message: "Successfully connected to GitHub Copilot", + } + + // Test enterprise endpoint first when configured. + // Note: /enterprises/{ent}/copilot/billing does not exist — use /billing/seats instead. + if hasEnterprise { + entSlug := strings.TrimSpace(connection.Enterprise) + seatsPath := copilotAPIPath("enterprises", entSlug, "copilot/billing/seats") + entSummary, entErr := fetchSeatsSummary(apiClient, seatsPath) + if entErr != nil { + return nil, entErr + } + result.Enterprise = entSlug + result.PlanType = "enterprise" + result.TotalSeats = entSummary.TotalSeats + } + + // Test org endpoint when configured. + if hasOrg { + orgSummary, orgErr := fetchBillingSummary(apiClient, copilotAPIPath("orgs", connection.Organization, "copilot/billing")) + if orgErr != nil { + return nil, orgErr + } + organization := orgSummary.Organization + if organization == "" { + organization = connection.Organization + } + result.Organization = organization + + // When enterprise is not set, use org-level data for the result. + if !hasEnterprise { + result.PlanType = orgSummary.PlanType + result.TotalSeats = orgSummary.TotalSeats + activeSeats := orgSummary.ActiveSeats + if activeSeats == 0 && orgSummary.ActiveThisCycle > 0 { + activeSeats = orgSummary.ActiveThisCycle + } + result.ActiveSeats = activeSeats + } + } + + return result, nil +} + +// fetchBillingSummary calls a billing endpoint and returns the parsed summary or error. +func fetchBillingSummary(apiClient *helper.ApiClient, path string) (*copilotBillingSummary, errors.Error) { + res, err := apiClient.Get(path, nil, nil) + if err != nil { + return nil, err + } + + if res.StatusCode >= 400 { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Convert(readErr) + } + return nil, buildGitHubApiError(res.StatusCode, path, body, res.Header.Get("Retry-After")) + } + + summary := &copilotBillingSummary{} + if err := helper.UnmarshalResponse(res, summary); err != nil { + return nil, err + } + return summary, nil +} + +// enterpriseSeatsSummary represents the top-level response from /copilot/billing/seats. +type enterpriseSeatsSummary struct { + TotalSeats int `json:"total_seats"` +} + +// fetchSeatsSummary calls the seats endpoint and returns the total seat count. +func fetchSeatsSummary(apiClient *helper.ApiClient, path string) (*enterpriseSeatsSummary, errors.Error) { + res, err := apiClient.Get(path, nil, nil) + if err != nil { + return nil, err + } + + if res.StatusCode >= 400 { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Convert(readErr) + } + return nil, buildGitHubApiError(res.StatusCode, path, body, res.Header.Get("Retry-After")) + } + + summary := &enterpriseSeatsSummary{} + if err := helper.UnmarshalResponse(res, summary); err != nil { + return nil, err + } + return summary, nil +} + +func buildGitHubApiError(status int, resource string, body []byte, retryAfter string) errors.Error { + type githubError struct { + Message string `json:"message"` + } + + msg := strings.TrimSpace(string(body)) + if len(body) > 0 { + errPayload := &githubError{} + if jsonErr := json.Unmarshal(body, errPayload); jsonErr == nil && errPayload.Message != "" { + msg = errPayload.Message + } + } + + var prefix string + switch status { + case http.StatusForbidden: + prefix = "GitHub returned 403 Forbidden. Ensure the PAT includes manage_billing:copilot and the resource has Copilot access." + case http.StatusNotFound: + prefix = fmt.Sprintf("GitHub returned 404 Not Found for '%s'. Verify the organization/enterprise slug and Copilot availability.", resource) + case http.StatusUnprocessableEntity: + prefix = "GitHub returned 422 Unprocessable Entity. Enable Copilot metrics before testing." + case http.StatusTooManyRequests: + prefix = "GitHub rate limited the request (429). Respect Retry-After guidance before retrying." + default: + prefix = fmt.Sprintf("GitHub API request failed with status %d.", status) + } + + if retryAfter != "" { + if seconds, err := strconv.Atoi(retryAfter); err == nil { + prefix = fmt.Sprintf("%s Retry after %d seconds.", prefix, seconds) + } else if delay, err := http.ParseTime(retryAfter); err == nil { + seconds := int(time.Until(delay).Seconds()) + if seconds > 0 { + prefix = fmt.Sprintf("%s Retry after %d seconds.", prefix, seconds) + } + } else { + prefix = fmt.Sprintf("%s Retry-After: %s.", prefix, retryAfter) + } + } + + if msg != "" { + prefix = fmt.Sprintf("%s Details: %s", prefix, msg) + } + + return errors.HttpStatus(status).New(strings.TrimSpace(prefix)) +} diff --git a/backend/plugins/gh-copilot/service/connection_test_helper_test.go b/backend/plugins/gh-copilot/service/connection_test_helper_test.go new file mode 100644 index 00000000000..a50742fc9ee --- /dev/null +++ b/backend/plugins/gh-copilot/service/connection_test_helper_test.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildGitHubApiError_Forbidden(t *testing.T) { + err := buildGitHubApiError(http.StatusForbidden, "octodemo", []byte(`{"message":"Resource not accessible"}`), "") + assert.Equal(t, http.StatusForbidden, err.GetType().GetHttpCode()) + assert.Contains(t, err.Error(), "Forbidden") + assert.Contains(t, err.Error(), "manage_billing:copilot") +} + +func TestBuildGitHubApiError_NotFound(t *testing.T) { + err := buildGitHubApiError(http.StatusNotFound, "octodemo", []byte(`{"message":"Not Found"}`), "") + assert.Equal(t, http.StatusNotFound, err.GetType().GetHttpCode()) + assert.Contains(t, err.Error(), "octodemo") +} + +func TestBuildGitHubApiError_UnprocessableEntity(t *testing.T) { + err := buildGitHubApiError(http.StatusUnprocessableEntity, "octodemo", []byte(`{"message":"Metrics disabled"}`), "") + assert.Equal(t, http.StatusUnprocessableEntity, err.GetType().GetHttpCode()) + assert.Contains(t, err.Error(), "Unprocessable") + assert.Contains(t, err.Error(), "Metrics disabled") +} + +func TestBuildGitHubApiError_TooManyRequests(t *testing.T) { + err := buildGitHubApiError(http.StatusTooManyRequests, "octodemo", []byte(`{"message":"Slow down"}`), "60") + assert.Equal(t, http.StatusTooManyRequests, err.GetType().GetHttpCode()) + assert.Contains(t, err.Error(), "429") + assert.Contains(t, err.Error(), "60 seconds") +} diff --git a/backend/plugins/gh-copilot/tasks/api_client.go b/backend/plugins/gh-copilot/tasks/api_client.go new file mode 100644 index 00000000000..1a58eb65b2e --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_client.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +const gitHubApiVersion = "2026-03-10" + +type nowFunc func() time.Time +type sleepFunc func(time.Duration) + +func handleGitHubRetryAfter(res *http.Response, logger log.Logger, now nowFunc, sleep sleepFunc) errors.Error { + if res == nil { + return nil + } + if res.StatusCode != http.StatusTooManyRequests { + return nil + } + + if now == nil { + now = time.Now + } + if sleep == nil { + sleep = time.Sleep + } + + wait := parseRetryAfter(res.Header.Get("Retry-After"), now().UTC()) + if wait > 0 { + if logger != nil { + logger.Warn(nil, "GitHub returned 429; sleeping %s per Retry-After", wait.String()) + } + sleep(wait) + } + // Return an error so the async client will retry. + return errors.HttpStatus(http.StatusTooManyRequests).New("GitHub rate limited the request") +} + +func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GhCopilotConnection) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + apiClient.SetHeaders(map[string]string{ + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": gitHubApiVersion, + }) + + rateLimiter := &helper.ApiRateLimitCalculator{UserRateLimitPerHour: connection.RateLimitPerHour} + asyncClient, err := helper.CreateAsyncApiClient(taskCtx, apiClient, rateLimiter) + if err != nil { + return nil, err + } + + // Ensure we respect GitHub Retry-After on 429s before retrying. + apiClient.SetAfterFunction(func(res *http.Response) errors.Error { + return handleGitHubRetryAfter(res, taskCtx.GetLogger(), time.Now, time.Sleep) + }) + + return asyncClient, nil +} diff --git a/backend/plugins/gh-copilot/tasks/api_client_test.go b/backend/plugins/gh-copilot/tasks/api_client_test.go new file mode 100644 index 00000000000..51485ad2ed0 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_client_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/http" + "testing" + "time" + + "github.com/apache/incubator-devlake/core/log" + "github.com/stretchr/testify/require" +) + +type captureLogger struct { + warnings []string +} + +func (l *captureLogger) IsLevelEnabled(level log.LogLevel) bool { return true } +func (l *captureLogger) Printf(format string, a ...interface{}) {} +func (l *captureLogger) Log(level log.LogLevel, format string, a ...interface{}) { +} +func (l *captureLogger) Debug(format string, a ...interface{}) {} +func (l *captureLogger) Info(format string, a ...interface{}) {} +func (l *captureLogger) Warn(err error, format string, a ...interface{}) { + l.warnings = append(l.warnings, fmt.Sprintf(format, a...)) +} +func (l *captureLogger) Error(err error, format string, a ...interface{}) {} +func (l *captureLogger) Nested(name string) log.Logger { return l } +func (l *captureLogger) GetConfig() *log.LoggerConfig { return &log.LoggerConfig{} } +func (l *captureLogger) SetStream(config *log.LoggerStreamConfig) {} + +func TestHandleGitHubRetryAfterSleepsAndReturnsError(t *testing.T) { + res := &http.Response{StatusCode: http.StatusTooManyRequests, Header: http.Header{}} + res.Header.Set("Retry-After", "10") + + logger := &captureLogger{} + var slept time.Duration + + err := handleGitHubRetryAfter( + res, + logger, + func() time.Time { return time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) }, + func(d time.Duration) { slept = d }, + ) + + require.NotNil(t, err) + require.Equal(t, 10*time.Second, slept) + require.Len(t, logger.warnings, 1) + require.Contains(t, logger.warnings[0], "sleeping") +} + +func TestHandleGitHubRetryAfterNoopOnNon429(t *testing.T) { + res := &http.Response{StatusCode: http.StatusOK, Header: http.Header{}} + var slept time.Duration + logger := &captureLogger{} + + err := handleGitHubRetryAfter(res, logger, nil, func(d time.Duration) { slept = d }) + require.Nil(t, err) + require.Equal(t, time.Duration(0), slept) + require.Len(t, logger.warnings, 0) +} + +func TestGitHubApiVersionHeaderValue(t *testing.T) { + require.Equal(t, "2026-03-10", gitHubApiVersion, + "GitHub API version header was changed; verify the new version is supported and update this test") +} diff --git a/backend/plugins/gh-copilot/tasks/api_path.go b/backend/plugins/gh-copilot/tasks/api_path.go new file mode 100644 index 00000000000..0d177ee14be --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_path.go @@ -0,0 +1,28 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "net/url" + "strings" +) + +func copilotAPIPath(namespace, slug, resource string) string { + return fmt.Sprintf("%s/%s/%s", namespace, url.PathEscape(strings.TrimSpace(slug)), strings.TrimPrefix(resource, "/")) +} diff --git a/backend/plugins/gh-copilot/tasks/api_path_test.go b/backend/plugins/gh-copilot/tasks/api_path_test.go new file mode 100644 index 00000000000..b92420484f3 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/api_path_test.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCopilotAPIPathPreservesHyphenatedEnterpriseSlug(t *testing.T) { + path := copilotAPIPath("enterprises", "my-enterprise", "copilot/billing/seats") + + require.Equal(t, "enterprises/my-enterprise/copilot/billing/seats", path) +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go new file mode 100644 index 00000000000..e153d59c948 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_collector.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawEnterpriseMetricsTable = "copilot_enterprise_metrics" + +// dayInput is passed to each collector request via the Input iterator. +type dayInput struct { + Day string `json:"day"` +} + +// CollectEnterpriseMetrics collects enterprise-level daily Copilot usage reports. +// It iterates day-by-day using the enterprise-1-day report endpoint, downloads +// the report files from the returned links, and stores them as raw data. +func CollectEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + if !connection.HasEnterprise() { + taskCtx.GetLogger().Info("No enterprise configured, skipping enterprise metrics collection") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawEnterpriseMetricsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + start = clampDailyMetricsStartForBackfill(start, until) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/enterprise-1-day"), + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + return parseRawReportResponse(res, logger) + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +// dayIterator implements helper.Iterator to yield one dayInput per day in a range. +type dayIterator struct { + days []dayInput + idx int +} + +func newDayIterator(start, end time.Time) *dayIterator { + var days []dayInput + for d := start; !d.After(end); d = d.AddDate(0, 0, 1) { + days = append(days, dayInput{Day: d.Format("2006-01-02")}) + } + return &dayIterator{days: days} +} + +func (it *dayIterator) HasNext() bool { + return it.idx < len(it.days) +} + +func (it *dayIterator) Fetch() (interface{}, errors.Error) { + if it.idx >= len(it.days) { + return nil, nil + } + day := it.days[it.idx] + it.idx++ + return &day, nil +} + +func (it *dayIterator) Close() errors.Error { + return nil +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go new file mode 100644 index 00000000000..e98a3c4f0e5 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go @@ -0,0 +1,369 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// --- Enterprise report JSON structures --- + +type enterpriseDayTotal struct { + Day string `json:"day"` + EnterpriseId string `json:"enterprise_id"` + DailyActiveUsers int `json:"daily_active_users"` + WeeklyActiveUsers int `json:"weekly_active_users"` + MonthlyActiveUsers int `json:"monthly_active_users"` + MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` + MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + DailyActiveCliUsers int `json:"daily_active_cli_users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"daily_active_copilot_code_review_users"` + DailyPassiveCopilotCodeReviewUsers int `json:"daily_passive_copilot_code_review_users"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weekly_active_copilot_code_review_users"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weekly_passive_copilot_code_review_users"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthly_active_copilot_code_review_users"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthly_passive_copilot_code_review_users"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chat_panel_agent_mode"` + ChatPanelAskMode int `json:"chat_panel_ask_mode"` + ChatPanelCustomMode int `json:"chat_panel_custom_mode"` + ChatPanelEditMode int `json:"chat_panel_edit_mode"` + ChatPanelPlanMode int `json:"chat_panel_plan_mode"` + ChatPanelUnknownMode int `json:"chat_panel_unknown_mode"` + + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` + TotalsByIde []totalsByIde `json:"totals_by_ide"` + TotalsByFeature []totalsByFeature `json:"totals_by_feature"` + TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` + TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` + TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` + PullRequests *pullRequestStats `json:"pull_requests"` +} + +type totalsByIde struct { + Ide string `json:"ide"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` +} + +type totalsByFeature struct { + Feature string `json:"feature"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` +} + +type totalsByLangFeature struct { + Language string `json:"language"` + Feature string `json:"feature"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` +} + +type totalsByLangModel struct { + Language string `json:"language"` + Model string `json:"model"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` +} + +type pullRequestStats struct { + TotalReviewed int `json:"total_reviewed"` + TotalCreated int `json:"total_created"` + TotalMerged int `json:"total_merged"` + MedianMinutesToMerge float64 `json:"median_minutes_to_merge"` + TotalSuggestions int `json:"total_suggestions"` + TotalAppliedSuggestions int `json:"total_applied_suggestions"` + TotalCreatedByCopilot int `json:"total_created_by_copilot"` + TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalMergedCreatedByCopilot int `json:"total_merged_created_by_copilot"` + TotalMergedReviewedByCopilot int `json:"total_merged_reviewed_by_copilot"` + MedianMinToMergeCopilotAuthored float64 `json:"median_minutes_to_merge_copilot_authored"` + MedianMinToMergeCopilotReviewed float64 `json:"median_minutes_to_merge_copilot_reviewed"` + TotalCopilotSuggestions int `json:"total_copilot_suggestions"` + TotalCopilotAppliedSuggestions int `json:"total_copilot_applied_suggestions"` +} + +type totalsByCli struct { + SessionCount int `json:"session_count"` + RequestCount int `json:"request_count"` + PromptCount int `json:"prompt_count"` + TokenUsage *cliTokens `json:"token_usage"` +} + +type cliTokens struct { + OutputTokensSum int `json:"output_tokens_sum"` + PromptTokensSum int `json:"prompt_tokens_sum"` +} + +type totalsByModelFeature struct { + Model string `json:"model"` + Feature string `json:"feature"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` +} + +// ExtractEnterpriseMetrics parses enterprise report JSON and extracts to tool-layer tables. +func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + if !connection.HasEnterprise() { + taskCtx.GetLogger().Info("No enterprise configured, skipping enterprise metrics extraction") + return nil + } + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawEnterpriseMetricsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + // The API returns a flat enterpriseDayTotal object per raw row, not a wrapper. + var dt enterpriseDayTotal + if err := errors.Convert(json.Unmarshal(row.Data, &dt)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", dt.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in enterprise report") + } + + var results []interface{} + + // Main daily metrics + dailyMetrics := &models.GhCopilotEnterpriseDailyMetrics{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + EnterpriseId: dt.EnterpriseId, + DailyActiveUsers: dt.DailyActiveUsers, + WeeklyActiveUsers: dt.WeeklyActiveUsers, + MonthlyActiveUsers: dt.MonthlyActiveUsers, + MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, + MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, + CodeGenerationActivityCount: dt.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: dt.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: dt.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: dt.LocSuggestedToDeleteSum, + LocAddedSum: dt.LocAddedSum, + LocDeletedSum: dt.LocDeletedSum, + }, + } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } + if dt.PullRequests != nil { + dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed + dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions + dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot + dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions + } + results = append(results, dailyMetrics) + + // By IDE + for _, ide := range dt.TotalsByIde { + results = append(results, &models.GhCopilotMetricsByIde{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Ide: ide.Ide, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: ide.UserInitiatedInteractionCount, + CodeGenerationActivityCount: ide.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: ide.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: ide.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: ide.LocSuggestedToDeleteSum, + LocAddedSum: ide.LocAddedSum, + LocDeletedSum: ide.LocDeletedSum, + }, + }) + } + + // By Feature + for _, f := range dt.TotalsByFeature { + results = append(results, &models.GhCopilotMetricsByFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Feature: f.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: f.UserInitiatedInteractionCount, + CodeGenerationActivityCount: f.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: f.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: f.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: f.LocSuggestedToDeleteSum, + LocAddedSum: f.LocAddedSum, + LocDeletedSum: f.LocDeletedSum, + }, + }) + } + + // By Language+Feature + for _, lf := range dt.TotalsByLanguageFeature { + results = append(results, &models.GhCopilotMetricsByLanguageFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Language: lf.Language, + Feature: lf.Feature, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lf.LocSuggestedToDeleteSum, + LocAddedSum: lf.LocAddedSum, + LocDeletedSum: lf.LocDeletedSum, + }, + }) + } + + // By Language+Model + for _, lm := range dt.TotalsByLanguageModel { + results = append(results, &models.GhCopilotMetricsByLanguageModel{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Language: lm.Language, + Model: lm.Model, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lm.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lm.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lm.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lm.LocSuggestedToDeleteSum, + LocAddedSum: lm.LocAddedSum, + LocDeletedSum: lm.LocDeletedSum, + }, + }) + } + + // By Model+Feature + for _, mf := range dt.TotalsByModelFeature { + results = append(results, &models.GhCopilotMetricsByModelFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Model: mf.Model, + Feature: mf.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: mf.UserInitiatedInteractionCount, + CodeGenerationActivityCount: mf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: mf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: mf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: mf.LocSuggestedToDeleteSum, + LocAddedSum: mf.LocAddedSum, + LocDeletedSum: mf.LocDeletedSum, + }, + }) + } + + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/github_errors.go b/backend/plugins/gh-copilot/tasks/github_errors.go new file mode 100644 index 00000000000..2280013307b --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/github_errors.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/apache/incubator-devlake/core/errors" +) + +type githubErrorPayload struct { + Message string `json:"message"` +} + +func buildGitHubApiError(status int, organization string, body []byte, retryAfter string) errors.Error { + msg := strings.TrimSpace(string(body)) + if len(body) > 0 { + payload := &githubErrorPayload{} + if err := json.Unmarshal(body, payload); err == nil && payload.Message != "" { + msg = payload.Message + } + } + + var prefix string + switch status { + case http.StatusForbidden: + prefix = "GitHub returned 403 Forbidden. Ensure the PAT includes manage_billing:copilot and the organization has Copilot access." + case http.StatusNotFound: + prefix = fmt.Sprintf("GitHub returned 404 Not Found for organization '%s'. Verify the organization slug and Copilot availability.", organization) + case http.StatusUnprocessableEntity: + prefix = "GitHub returned 422 Unprocessable Entity. Enable Copilot metrics for the organization before running collection." + case http.StatusTooManyRequests: + prefix = "GitHub rate limited the request (429). Respect Retry-After guidance before retrying." + default: + prefix = fmt.Sprintf("GitHub API request failed with status %d.", status) + } + + if retryAfter != "" { + prefix = fmt.Sprintf("%s Retry-After: %s.", prefix, retryAfter) + } + if msg != "" { + prefix = fmt.Sprintf("%s Details: %s", prefix, msg) + } + return errors.HttpStatus(status).New(strings.TrimSpace(prefix)) +} diff --git a/backend/plugins/gh-copilot/tasks/github_errors_test.go b/backend/plugins/gh-copilot/tasks/github_errors_test.go new file mode 100644 index 00000000000..ef817b77b79 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/github_errors_test.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBuildGitHubApiErrorExtractsMessageFromJson(t *testing.T) { + err := buildGitHubApiError(http.StatusForbidden, "octodemo", []byte(`{"message":"nope"}`), "") + require.Contains(t, err.Error(), "403") + require.Contains(t, err.Error(), "Details: nope") +} + +func TestBuildGitHubApiErrorIncludesRetryAfter(t *testing.T) { + err := buildGitHubApiError(http.StatusTooManyRequests, "octodemo", []byte(""), "120") + require.Contains(t, err.Error(), "429") + require.Contains(t, err.Error(), "Retry-After: 120") +} + +func TestBuildGitHubApiErrorStatusSpecificPrefixes(t *testing.T) { + tests := []struct { + name string + status int + want string + }{ + {name: "403", status: http.StatusForbidden, want: "403 Forbidden"}, + {name: "404", status: http.StatusNotFound, want: "404 Not Found"}, + {name: "422", status: http.StatusUnprocessableEntity, want: "422 Unprocessable Entity"}, + {name: "429", status: http.StatusTooManyRequests, want: "429"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := buildGitHubApiError(tt.status, "octodemo", []byte("boom"), "") + require.Contains(t, err.Error(), tt.want) + }) + } +} diff --git a/backend/plugins/gh-copilot/tasks/metrics_collector_test.go b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go new file mode 100644 index 00000000000..d71d20d4440 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/metrics_collector_test.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "io" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestParseRetryAfterSeconds(t *testing.T) { + wait := parseRetryAfter("10", time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)) + require.Equal(t, 10*time.Second, wait) +} + +func TestParseRetryAfterHttpDate(t *testing.T) { + now := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + value := now.Add(5 * time.Second).Format(http.TimeFormat) + wait := parseRetryAfter(value, now) + require.True(t, wait >= 4*time.Second && wait <= 6*time.Second) +} + +func TestComputeReportDateRangeDefaultLookback(t *testing.T) { + now := time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + start, until := computeReportDateRange(now, nil) + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2024, 1, 11, 0, 0, 0, 0, time.UTC), start) +} + +func TestComputeReportDateRangeUsesSince(t *testing.T) { + // since is far enough in the past that the lookback buffer doesn't apply. + now := time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + since := time.Date(2025, 1, 3, 12, 0, 0, 0, time.UTC) + start, until := computeReportDateRange(now, &since) + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2025, 1, 3, 0, 0, 0, 0, time.UTC), start) +} + +func TestComputeReportDateRangeClampsToLookback(t *testing.T) { + now := time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + since := time.Date(2024, 6, 24, 12, 0, 0, 0, time.UTC) + start, until := computeReportDateRange(now, &since) + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2024, 6, 24, 0, 0, 0, 0, time.UTC), start) +} + +func TestComputeReportDateRangeClampsFutureSince(t *testing.T) { + // Future since is clamped to until, then the lookback buffer applies. + now := time.Date(2025, 1, 10, 12, 0, 0, 0, time.UTC) + since := now.Add(24 * time.Hour) + start, until := computeReportDateRange(now, &since) + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2025, 1, 7, 0, 0, 0, 0, time.UTC), start) +} + +func TestComputeReportDateRangeLookbackBuffer(t *testing.T) { + // since is yesterday: without the buffer we'd only request 1 day (yesterday). + // With the buffer we look back reportLookbackDays days to retry any 404'd days. + now := time.Date(2025, 1, 10, 0, 0, 0, 0, time.UTC) // midnight run + since := time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC) // LatestSuccessStart from previous midnight run + start, until := computeReportDateRange(now, &since) + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2025, 1, 7, 0, 0, 0, 0, time.UTC), start) +} + +func TestClampDailyMetricsStartForBackfillRecentStart(t *testing.T) { + until := time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC) + start := time.Date(2025, 1, 7, 0, 0, 0, 0, time.UTC) + + clamped := clampDailyMetricsStartForBackfill(start, until) + require.Equal(t, time.Date(2025, 1, 6, 0, 0, 0, 0, time.UTC), clamped) +} + +func TestClampDailyMetricsStartForBackfillKeepsOlderStart(t *testing.T) { + until := time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC) + start := time.Date(2025, 1, 3, 0, 0, 0, 0, time.UTC) + + clamped := clampDailyMetricsStartForBackfill(start, until) + require.Equal(t, start, clamped) +} + +func TestUserMetricsDateRangeAppliesFourDayBackfillWindow(t *testing.T) { + now := time.Date(2025, 1, 10, 0, 0, 0, 0, time.UTC) + since := time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC) + + start, until := computeReportDateRange(now, &since) + start = clampDailyMetricsStartForBackfill(start, until) + + require.Equal(t, time.Date(2025, 1, 9, 0, 0, 0, 0, time.UTC), until) + require.Equal(t, time.Date(2025, 1, 6, 0, 0, 0, 0, time.UTC), start) +} + +func TestParseReportMetadataResponseNoContent(t *testing.T) { + res := &http.Response{ + StatusCode: http.StatusNoContent, + Body: io.NopCloser(bytes.NewReader(nil)), + } + + meta, err := parseReportMetadataResponse(res, nil) + require.NoError(t, err) + require.Nil(t, meta) +} + +func TestParseReportMetadataResponseEmptyBody(t *testing.T) { + res := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader(nil)), + } + + meta, err := parseReportMetadataResponse(res, nil) + require.NoError(t, err) + require.Nil(t, meta) +} + +func TestParseReportMetadataResponseEmptyString(t *testing.T) { + res := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte(`""`))), + } + + meta, err := parseReportMetadataResponse(res, nil) + require.NoError(t, err) + require.Nil(t, meta) +} + +func TestIsEmptyReport(t *testing.T) { + tests := []struct { + name string + body []byte + want bool + }{ + {"empty JSON string", []byte(`""`), true}, + {"null", []byte("null"), true}, + {"empty body", []byte{}, true}, + {"whitespace only", []byte(" "), true}, + {"padded empty string", []byte(` "" `), true}, + {"valid metadata", []byte(`{"download_links":["https://example.com/report.json"],"report_day":"2026-03-19"}`), false}, + {"valid metadata empty links", []byte(`{"download_links":[],"report_day":"2026-03-19"}`), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, isEmptyReport(tt.body)) + }) + } +} diff --git a/backend/plugins/gh-copilot/tasks/metrics_extractor.go b/backend/plugins/gh-copilot/tasks/metrics_extractor.go new file mode 100644 index 00000000000..d89eababde6 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/metrics_extractor.go @@ -0,0 +1,270 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// Seat response structs (used by seat_extractor.go) + +type copilotSeatResponse struct { + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + PlanType string `json:"plan_type"` + PendingCancellationDate *string `json:"pending_cancellation_date"` + LastAuthenticatedAt *string `json:"last_authenticated_at"` + LastActivityAt *string `json:"last_activity_at"` + LastActivityEditor string `json:"last_activity_editor"` + Assignee copilotAssignee `json:"assignee"` + AssigningTeam *copilotTeam `json:"assigning_team"` +} + +type copilotAssignee struct { + Login string `json:"login"` + Id int64 `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + Email string `json:"email"` +} + +type copilotTeam struct { + Id int64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` +} + +// ExtractOrgMetrics parses org report data from the new report download API. +// The org report uses the same flat format as enterprise reports (day, totals_by_*). +// It writes to the same unified tables as ExtractEnterpriseMetrics so the +// Grafana dashboard works identically for org-only and enterprise connections. +func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + if connection.Organization == "" { + taskCtx.GetLogger().Info("No organization configured, skipping org metrics extraction") + return nil + } + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawOrgMetricsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var dt enterpriseDayTotal + if err := errors.Convert(json.Unmarshal(row.Data, &dt)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", dt.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in org report") + } + + var results []interface{} + + // Main daily metrics — same model as enterprise extractor + dailyMetrics := &models.GhCopilotEnterpriseDailyMetrics{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + EnterpriseId: "", // org-level, no enterprise + DailyActiveUsers: dt.DailyActiveUsers, + WeeklyActiveUsers: dt.WeeklyActiveUsers, + MonthlyActiveUsers: dt.MonthlyActiveUsers, + MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, + MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, + CodeGenerationActivityCount: dt.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: dt.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: dt.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: dt.LocSuggestedToDeleteSum, + LocAddedSum: dt.LocAddedSum, + LocDeletedSum: dt.LocDeletedSum, + }, + } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } + if dt.PullRequests != nil { + dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed + dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions + dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot + dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions + } + results = append(results, dailyMetrics) + + // By IDE + for _, ide := range dt.TotalsByIde { + results = append(results, &models.GhCopilotMetricsByIde{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Ide: ide.Ide, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: ide.UserInitiatedInteractionCount, + CodeGenerationActivityCount: ide.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: ide.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: ide.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: ide.LocSuggestedToDeleteSum, + LocAddedSum: ide.LocAddedSum, + LocDeletedSum: ide.LocDeletedSum, + }, + }) + } + + // By Feature + for _, f := range dt.TotalsByFeature { + results = append(results, &models.GhCopilotMetricsByFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Feature: f.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: f.UserInitiatedInteractionCount, + CodeGenerationActivityCount: f.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: f.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: f.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: f.LocSuggestedToDeleteSum, + LocAddedSum: f.LocAddedSum, + LocDeletedSum: f.LocDeletedSum, + }, + }) + } + + // By Language+Feature + for _, lf := range dt.TotalsByLanguageFeature { + results = append(results, &models.GhCopilotMetricsByLanguageFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Language: lf.Language, + Feature: lf.Feature, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lf.LocSuggestedToDeleteSum, + LocAddedSum: lf.LocAddedSum, + LocDeletedSum: lf.LocDeletedSum, + }, + }) + } + + // By Language+Model + for _, lm := range dt.TotalsByLanguageModel { + results = append(results, &models.GhCopilotMetricsByLanguageModel{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Language: lm.Language, + Model: lm.Model, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lm.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lm.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lm.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lm.LocSuggestedToDeleteSum, + LocAddedSum: lm.LocAddedSum, + LocDeletedSum: lm.LocDeletedSum, + }, + }) + } + + // By Model+Feature + for _, mf := range dt.TotalsByModelFeature { + results = append(results, &models.GhCopilotMetricsByModelFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + Model: mf.Model, + Feature: mf.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: mf.UserInitiatedInteractionCount, + CodeGenerationActivityCount: mf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: mf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: mf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: mf.LocSuggestedToDeleteSum, + LocAddedSum: mf.LocAddedSum, + LocDeletedSum: mf.LocDeletedSum, + }, + }) + } + + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/options.go b/backend/plugins/gh-copilot/tasks/options.go new file mode 100644 index 00000000000..705c0370571 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/options.go @@ -0,0 +1,24 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +// GhCopilotOptions defines task-level options passed from pipeline plans. +type GhCopilotOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeId string `mapstructure:"scopeId" json:"scopeId"` +} diff --git a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go new file mode 100644 index 00000000000..c75ce7b6ce6 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go @@ -0,0 +1,133 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawOrgMetricsTable = "copilot_org_metrics" + +// CollectOrgMetrics collects organization-level daily Copilot usage reports +// using the new report download API. Replaces the deprecated /orgs/{org}/copilot/metrics endpoint. +func CollectOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + if connection.Organization == "" { + taskCtx.GetLogger().Info("No organization configured, skipping org metrics collection") + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawOrgMetricsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + start = clampDailyMetricsStartForBackfill(start, until) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/organization-1-day"), + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + snippet := string(body) + if len(snippet) > 200 { + snippet = snippet[:200] + } + logger.Error(jsonErr, "failed to parse report metadata, body=%s", snippet) + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + if len(meta.DownloadLinks) == 0 { + logger.Info("No download links for report day=%s, skipping", meta.ReportDay) + return nil, nil + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue // blob not found, skip + } + results = append(results, json.RawMessage(reportBody)) + } + return results, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go new file mode 100644 index 00000000000..3c7e5b1eeb9 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/core/plugin" + +// GetSubTaskMetas returns the ordered list of Copilot subtasks. +func GetSubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + // Collectors + CollectOrgMetricsMeta, + CollectCopilotSeatAssignmentsMeta, + CollectEnterpriseMetricsMeta, + CollectUserMetricsMeta, + CollectUserTeamsMeta, + // Extractors + ExtractSeatsMeta, + ExtractOrgMetricsMeta, + ExtractEnterpriseMetricsMeta, + ExtractUserMetricsMeta, + ExtractUserTeamsMeta, + } +} diff --git a/backend/plugins/gh-copilot/tasks/report_download_helper.go b/backend/plugins/gh-copilot/tasks/report_download_helper.go new file mode 100644 index 00000000000..236b039889b --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/report_download_helper.go @@ -0,0 +1,301 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// reportMaxDays is the maximum historical window the new report API supports (1 year). +const reportMaxDays = 365 + +// reportLookbackDays: extra days rewound from 'until' on incremental runs. +// GitHub reports are generated hours after midnight, so a midnight run gets 404 for the previous +// day. Without this buffer, 'LatestSuccessStart' advances past the missed day permanently. +const reportLookbackDays = 2 + +// dailyMetricsTrailingBackfillDays extends retries for delayed daily report generation. +const dailyMetricsTrailingBackfillDays = 4 + +// copilotRawParams identifies a set of raw data records for a given connection/scope. +type copilotRawParams struct { + ConnectionId uint64 + ScopeId string + Organization string + Endpoint string +} + +func (p copilotRawParams) GetParams() any { + return p +} + +func utcDate(t time.Time) time.Time { + y, m, d := t.UTC().Date() + return time.Date(y, m, d, 0, 0, 0, 0, time.UTC) +} + +// ignoreNoContent is an AfterResponse callback that skips 404 and 204 responses. +// The report API returns 404 when no report is available for a given day, +// and 204 (No Content) when data is not yet available or the org had fewer +// than 5 active Copilot users on that day. Both are normal and should not +// be treated as errors. +func ignoreNoContent(res *http.Response) errors.Error { + if res.StatusCode == http.StatusNotFound || res.StatusCode == http.StatusNoContent { + return helper.ErrIgnoreAndContinue + } + return nil +} + +func clampDailyMetricsStartForBackfill(start, until time.Time) time.Time { + trailingStart := until.AddDate(0, 0, -(dailyMetricsTrailingBackfillDays - 1)) + if start.After(trailingStart) { + return trailingStart + } + return start +} + +// isEmptyReport returns true when the GitHub API returned an HTTP 200 but the +// body carries no usable report data. For dates before Copilot usage data was +// available the API responds with "" (empty JSON string) instead of a 404. +func isEmptyReport(body []byte) bool { + b := bytes.TrimSpace(body) + return len(b) == 0 || string(b) == `""` || string(b) == "null" +} + +// reportMetadataResponse represents the JSON returned by the report metadata endpoints. +type reportMetadataResponse struct { + DownloadLinks []string `json:"download_links"` + ReportDay string `json:"report_day"` + // 28-day variants use start/end instead of a single day. + ReportStartDay string `json:"report_start_day"` + ReportEndDay string `json:"report_end_day"` +} + +func readReportMetadataBody(res *http.Response) ([]byte, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + return body, nil +} + +func logReportMetadataParseError(body []byte, err error, logger log.Logger) { + if logger == nil { + return + } + snippet := string(body) + if len(snippet) > 200 { + snippet = snippet[:200] + } + logger.Error(err, "failed to parse report metadata, body=%s", snippet) +} + +func reportMetadataRange(meta reportMetadataResponse) string { + if meta.ReportDay != "" { + return meta.ReportDay + } + if meta.ReportStartDay != "" && meta.ReportEndDay != "" { + return fmt.Sprintf("%s..%s", meta.ReportStartDay, meta.ReportEndDay) + } + return "" +} + +func logMissingDownloadLinks(meta reportMetadataResponse, logger log.Logger) { + if logger == nil || len(meta.DownloadLinks) != 0 { + return + } + reportRange := reportMetadataRange(meta) + if reportRange != "" { + logger.Info("No download links for report day=%s, skipping", reportRange) + return + } + logger.Info("No download links in report metadata, skipping") +} + +func parseReportMetadata(body []byte, logger log.Logger) (*reportMetadataResponse, errors.Error) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + if logger != nil { + logger.Info("Report metadata response was empty, skipping") + } + return nil, nil + } + + // Handle JSON-encoded empty string "" + if bytes.Equal(trimmed, []byte(`""`)) { + if logger != nil { + logger.Info("Report metadata response was empty string, skipping") + } + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(trimmed, &meta); jsonErr != nil { + logReportMetadataParseError(trimmed, jsonErr, logger) + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + logMissingDownloadLinks(meta, logger) + + return &meta, nil +} + +func parseReportMetadataResponse(res *http.Response, logger log.Logger) (*reportMetadataResponse, errors.Error) { + if res.StatusCode == http.StatusNoContent { + if logger != nil { + logger.Info("Report metadata not ready yet (204), skipping for now") + } + res.Body.Close() + return nil, nil + } + + body, readErr := readReportMetadataBody(res) + if readErr != nil { + return nil, readErr + } + + return parseReportMetadata(body, logger) +} + +func collectRawReportRecords(meta *reportMetadataResponse, logger log.Logger) ([]json.RawMessage, errors.Error) { + if len(meta.DownloadLinks) == 0 { + logger.Info("No download links for report day=%s, skipping", meta.ReportDay) + return nil, nil + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue + } + results = append(results, json.RawMessage(reportBody)) + } + return results, nil +} + +func parseRawReportResponse(res *http.Response, logger log.Logger) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta *reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + snippet := string(body) + if len(snippet) > 200 { + snippet = snippet[:200] + } + logger.Error(jsonErr, "failed to parse report metadata, body=%s", snippet) + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + meta, err := parseReportMetadataResponse(res, logger) + if err != nil || meta == nil { + return nil, err + } + + return collectRawReportRecords(meta, logger) +} + +// computeReportDateRange returns the range of dates to collect, clamped to the API max. +// When 'since' is set, 'start' is rewound to at least 'until - reportLookbackDays' +// so days that returned 404 (report not yet generated) are retried on subsequent runs. +func computeReportDateRange(now time.Time, since *time.Time) (start, until time.Time) { + until = utcDate(now).AddDate(0, 0, -1) // reports are available for the previous day + min := until.AddDate(0, 0, -(reportMaxDays - 1)) + start = min + if since != nil { + start = utcDate(*since) + if start.Before(min) { + start = min + } + if start.After(until) { + start = until + } + // Rewind 'start' by 'reportLookbackDays' so recently-missed days are retried. + if lookback := until.AddDate(0, 0, -reportLookbackDays); start.After(lookback) { + start = lookback + } + } + return start, until +} + +// downloadReport downloads a single report file from a signed URL and returns the raw body. +// Returns nil, nil when the blob is not found (404) — the caller should skip such reports. +func downloadReport(url string, logger log.Logger) ([]byte, errors.Error) { + resp, err := http.Get(url) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to download report file") + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + if logger != nil { + logger.Info("Report blob not found (404), skipping") + } + return nil, nil + } + + if resp.StatusCode >= 400 { + body, _ := io.ReadAll(resp.Body) + return nil, errors.Default.New(fmt.Sprintf("report download failed with status %d: %s", resp.StatusCode, string(body))) + } + + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report file body") + } + + if logger != nil { + logger.Info("Downloaded report file (%d bytes)", len(body)) + } + return body, nil +} + +// parseJSONL splits a JSONL (JSON Lines) byte slice into individual JSON messages. +// Each non-empty line is treated as a separate JSON object. +func parseJSONL(data []byte) ([]json.RawMessage, errors.Error) { + var results []json.RawMessage + lines := bytes.Split(data, []byte("\n")) + for _, line := range lines { + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + results = append(results, json.RawMessage(line)) + } + return results, nil +} diff --git a/backend/plugins/gh-copilot/tasks/retry_after.go b/backend/plugins/gh-copilot/tasks/retry_after.go new file mode 100644 index 00000000000..6b0a36b805e --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/retry_after.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + "strconv" + "time" +) + +func parseRetryAfter(value string, now time.Time) time.Duration { + if value == "" { + return 0 + } + if seconds, err := strconv.Atoi(value); err == nil { + if seconds <= 0 { + return 0 + } + return time.Duration(seconds) * time.Second + } + if t, err := http.ParseTime(value); err == nil { + wait := t.Sub(now) + if wait < 0 { + return 0 + } + return wait + } + return 0 +} diff --git a/backend/plugins/gh-copilot/tasks/seat_collector.go b/backend/plugins/gh-copilot/tasks/seat_collector.go new file mode 100644 index 00000000000..546a0e30abc --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/seat_collector.go @@ -0,0 +1,129 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawCopilotSeatsTable = "copilot_seats" + +func parseCopilotSeatsFromResponse(res *http.Response) ([]json.RawMessage, errors.Error) { + if res == nil { + return nil, errors.Default.New("res is nil") + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, errors.Default.Wrap(err, fmt.Sprintf("error reading response body of %s", res.Request.URL.String())) + } + + // Some GitHub endpoints return a top-level array, others return an object wrapper. + // Support both, returning the underlying array of seat objects. + var rawMessages []json.RawMessage + if jsonErr := json.Unmarshal(body, &rawMessages); jsonErr == nil { + return rawMessages, nil + } + + var wrapped struct { + Seats []json.RawMessage `json:"seats"` + } + if jsonErr := json.Unmarshal(body, &wrapped); jsonErr != nil { + return nil, errors.Default.Wrap(errors.Convert(jsonErr), fmt.Sprintf("error decoding response of %s: raw response: %s", res.Request.URL.String(), string(body))) + } + return wrapped.Seats, nil +} + +func CollectCopilotSeatAssignments(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawCopilotSeatsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + var urlTemplate string + switch { + case connection.HasEnterprise(): + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/billing/seats") + case connection.Organization != "": + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/billing/seats") + default: + taskCtx.GetLogger().Warn(nil, "skipping seat collection: no enterprise or organization configured on connection %d", connection.ID) + return nil + } + + perPage := 100 + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + PageSize: perPage, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + q := url.Values{} + q.Set("per_page", fmt.Sprintf("%d", reqData.Pager.Size)) + q.Set("page", fmt.Sprintf("%d", reqData.Pager.Page)) + return q, nil + }, + GetNextPageCustomData: func(prevReqData *helper.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + // Standard page/per_page pagination; nothing extra to carry between pages. + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + if res.StatusCode >= 400 { + body, _ := io.ReadAll(res.Body) + res.Body.Close() + return nil, buildGitHubApiError(res.StatusCode, connection.Organization, body, res.Header.Get("Retry-After")) + } + return parseCopilotSeatsFromResponse(res) + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/seat_collector_test.go b/backend/plugins/gh-copilot/tasks/seat_collector_test.go new file mode 100644 index 00000000000..5c01d041b86 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/seat_collector_test.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseCopilotSeatsFromResponse_WrappedObject(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://api.github.com/orgs/octodemo/copilot/billing/seats?page=1", nil) + res := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`{"total_seats":2,"seats":[{"assignee":{"login":"a"}},{"assignee":{"login":"b"}}]}`)), + Request: req, + } + + msgs, err := parseCopilotSeatsFromResponse(res) + require.NoError(t, err) + require.Len(t, msgs, 2) +} + +func TestParseCopilotSeatsFromResponse_Array(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "https://api.github.com/orgs/octodemo/copilot/billing/seats?page=1", nil) + res := &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(`[{"assignee":{"login":"a"}}]`)), + Request: req, + } + + msgs, err := parseCopilotSeatsFromResponse(res) + require.NoError(t, err) + require.Len(t, msgs, 1) +} diff --git a/backend/plugins/gh-copilot/tasks/seat_extractor.go b/backend/plugins/gh-copilot/tasks/seat_extractor.go new file mode 100644 index 00000000000..1a1b6b13518 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/seat_extractor.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// ExtractSeats parses raw seat assignment data into the GhCopilotSeat tool-layer model. +func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawCopilotSeatsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + seat := &copilotSeatResponse{} + if err := errors.Convert(json.Unmarshal(row.Data, seat)); err != nil { + return nil, err + } + + createdAt, parseErr := time.Parse(time.RFC3339, seat.CreatedAt) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid seat created_at") + } + updatedAt, parseErr := time.Parse(time.RFC3339, seat.UpdatedAt) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid seat updated_at") + } + + parseOptional := func(v *string) (*time.Time, errors.Error) { + if v == nil || *v == "" { + return nil, nil + } + if t, parseErr := time.Parse(time.RFC3339, *v); parseErr == nil { + return &t, nil + } + t, parseErr := time.Parse("2006-01-02", *v) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid timestamp") + } + return &t, nil + } + + lastAuth, err := parseOptional(seat.LastAuthenticatedAt) + if err != nil { + return nil, err + } + lastAct, err := parseOptional(seat.LastActivityAt) + if err != nil { + return nil, err + } + pendingCancel, err := parseOptional(seat.PendingCancellationDate) + if err != nil { + return nil, err + } + + toolSeat := &models.GhCopilotSeat{ + ConnectionId: data.Options.ConnectionId, + Organization: connection.Organization, + UserLogin: seat.Assignee.Login, + UserId: seat.Assignee.Id, + UserName: seat.Assignee.Name, + UserEmail: seat.Assignee.Email, + PlanType: seat.PlanType, + CreatedAt: createdAt, + LastActivityAt: lastAct, + LastActivityEditor: seat.LastActivityEditor, + LastAuthenticatedAt: lastAuth, + PendingCancellationDate: pendingCancel, + UpdatedAt: updatedAt, + } + if seat.AssigningTeam != nil { + toolSeat.AssigningTeamId = seat.AssigningTeam.Id + toolSeat.AssigningTeamName = seat.AssigningTeam.Name + toolSeat.AssigningTeamSlug = seat.AssigningTeam.Slug + } + + return []interface{}{toolSeat}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go new file mode 100644 index 00000000000..61ed5799525 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +var CollectOrgMetricsMeta = plugin.SubTaskMeta{ + Name: "collectOrgMetrics", + EntryPoint: CollectOrgMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot organization usage metrics reports", +} + +var CollectCopilotSeatAssignmentsMeta = plugin.SubTaskMeta{ + Name: "collectCopilotSeatAssignments", + EntryPoint: CollectCopilotSeatAssignments, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot seat assignments", +} + +var CollectEnterpriseMetricsMeta = plugin.SubTaskMeta{ + Name: "collectEnterpriseMetrics", + EntryPoint: CollectEnterpriseMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot enterprise usage metrics reports", +} + +var CollectUserMetricsMeta = plugin.SubTaskMeta{ + Name: "collectUserMetrics", + EntryPoint: CollectUserMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot enterprise user-level usage metrics reports", +} + +var CollectUserTeamsMeta = plugin.SubTaskMeta{ + Name: "collectUserTeams", + EntryPoint: CollectUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot user-team mappings from user-teams-1-day report", +} + +var ExtractOrgMetricsMeta = plugin.SubTaskMeta{ + Name: "extractOrgMetrics", + EntryPoint: ExtractOrgMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot org metrics into unified daily metrics tables", + Dependencies: []*plugin.SubTaskMeta{&CollectOrgMetricsMeta}, +} + +var ExtractSeatsMeta = plugin.SubTaskMeta{ + Name: "extractSeats", + EntryPoint: ExtractSeats, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot seat assignments into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectCopilotSeatAssignmentsMeta}, +} + +var ExtractEnterpriseMetricsMeta = plugin.SubTaskMeta{ + Name: "extractEnterpriseMetrics", + EntryPoint: ExtractEnterpriseMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot enterprise metrics into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectEnterpriseMetricsMeta}, +} + +var ExtractUserMetricsMeta = plugin.SubTaskMeta{ + Name: "extractUserMetrics", + EntryPoint: ExtractUserMetrics, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot user metrics into tool-layer tables", + Dependencies: []*plugin.SubTaskMeta{&CollectUserMetricsMeta}, +} + +var ExtractUserTeamsMeta = plugin.SubTaskMeta{ + Name: "extractUserTeams", + EntryPoint: ExtractUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot user-team mappings into tool-layer table", + Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/task_data.go b/backend/plugins/gh-copilot/tasks/task_data.go new file mode 100644 index 00000000000..59b92fe96e3 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/task_data.go @@ -0,0 +1,26 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import "github.com/apache/incubator-devlake/plugins/gh-copilot/models" + +// GhCopilotTaskData stores runtime dependencies for subtasks. +type GhCopilotTaskData struct { + Options *GhCopilotOptions + Connection *models.GhCopilotConnection +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_collector.go b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go new file mode 100644 index 00000000000..8198b48a624 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_metrics_collector.go @@ -0,0 +1,148 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawUserMetricsTable = "copilot_user_metrics" + +func collectUserMetricsRecords(meta *reportMetadataResponse, logger log.Logger) ([]json.RawMessage, errors.Error) { + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue // blob not found, skip + } + // Parse JSONL: split by newlines and return each non-empty line. + userRecords, parseErr := parseJSONL(reportBody) + if parseErr != nil { + return nil, parseErr + } + results = append(results, userRecords...) + } + return results, nil +} + +func parseUserMetricsReportResponse(res *http.Response, logger log.Logger) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta *reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + meta, err := parseReportMetadataResponse(res, logger) + if err != nil || meta == nil { + return nil, err + } + + return collectUserMetricsRecords(meta, logger) +} + +// CollectUserMetrics collects enterprise user-level daily Copilot usage reports. +// These reports are in JSONL format (one JSON object per line per user). +// Utilizes the enterprise or organization endpoints depending on connection configuration +func CollectUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + var urlTemplate string + + if connection.HasEnterprise() { + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/users-1-day") + } else if connection.Organization != "" { + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/users-1-day") + } else { + return nil + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserMetricsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + start = clampDailyMetricsStartForBackfill(start, until) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + return parseUserMetricsReportResponse(res, logger) + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go new file mode 100644 index 00000000000..72992194063 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -0,0 +1,265 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// --- User report JSONL structures (one line per user) --- + +type userDailyReport struct { + ReportStartDay string `json:"report_start_day"` + ReportEndDay string `json:"report_end_day"` + Day string `json:"day"` + OrganizationId string `json:"organization_id"` + EnterpriseId string `json:"enterprise_id"` + UserId int64 `json:"user_id"` + UserLogin string `json:"user_login"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` + CodeGenerationActivityCount int `json:"code_generation_activity_count"` + CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` + LocSuggestedToAddSum int `json:"loc_suggested_to_add_sum"` + LocSuggestedToDeleteSum int `json:"loc_suggested_to_delete_sum"` + LocAddedSum int `json:"loc_added_sum"` + LocDeletedSum int `json:"loc_deleted_sum"` + UsedAgent bool `json:"used_agent"` + UsedChat bool `json:"used_chat"` + UsedCli bool `json:"used_cli"` + UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` + UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` + TotalsByIde []userTotalsByIde `json:"totals_by_ide"` + TotalsByFeature []totalsByFeature `json:"totals_by_feature"` + TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` + TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` + TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` +} + +type userTotalsByIde struct { + totalsByIde + LastKnownPluginVersion *pluginVersion `json:"last_known_plugin_version"` + LastKnownIdeVersion *ideVersion `json:"last_known_ide_version"` +} + +type pluginVersion struct { + SampledAt string `json:"sampled_at"` + Plugin string `json:"plugin"` + PluginVersion string `json:"plugin_version"` +} + +type ideVersion struct { + SampledAt string `json:"sampled_at"` + IdeVersion string `json:"ide_version"` +} + +// ExtractUserMetrics parses user report JSONL records and extracts to tool-layer tables. +func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserMetricsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var u userDailyReport + if err := errors.Convert(json.Unmarshal(row.Data, &u)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", u.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in user report") + } + + var results []interface{} + + // Main user daily metrics + userMetrics := &models.GhCopilotUserDailyMetrics{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + OrganizationId: u.OrganizationId, + EnterpriseId: u.EnterpriseId, + UserLogin: u.UserLogin, + UsedAgent: u.UsedAgent, + UsedChat: u.UsedChat, + UsedCli: u.UsedCli, + UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, + UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, + CodeGenerationActivityCount: u.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: u.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: u.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: u.LocSuggestedToDeleteSum, + LocAddedSum: u.LocAddedSum, + LocDeletedSum: u.LocDeletedSum, + }, + } + if u.TotalsByCli != nil { + userMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: u.TotalsByCli.SessionCount, + CliRequestCount: u.TotalsByCli.RequestCount, + CliPromptCount: u.TotalsByCli.PromptCount, + } + if u.TotalsByCli.TokenUsage != nil { + userMetrics.CopilotCliMetrics.CliOutputTokenSum = u.TotalsByCli.TokenUsage.OutputTokensSum + userMetrics.CopilotCliMetrics.CliPromptTokenSum = u.TotalsByCli.TokenUsage.PromptTokensSum + } + } + results = append(results, userMetrics) + + // User by IDE + for _, ide := range u.TotalsByIde { + rec := &models.GhCopilotUserMetricsByIde{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + Ide: ide.Ide, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: ide.UserInitiatedInteractionCount, + CodeGenerationActivityCount: ide.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: ide.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: ide.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: ide.LocSuggestedToDeleteSum, + LocAddedSum: ide.LocAddedSum, + LocDeletedSum: ide.LocDeletedSum, + }, + } + if ide.LastKnownPluginVersion != nil { + rec.LastKnownPluginName = ide.LastKnownPluginVersion.Plugin + rec.LastKnownPluginVersion = ide.LastKnownPluginVersion.PluginVersion + } + if ide.LastKnownIdeVersion != nil { + rec.LastKnownIdeVersion = ide.LastKnownIdeVersion.IdeVersion + } + results = append(results, rec) + } + + // User by Feature + for _, f := range u.TotalsByFeature { + results = append(results, &models.GhCopilotUserMetricsByFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + Feature: f.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: f.UserInitiatedInteractionCount, + CodeGenerationActivityCount: f.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: f.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: f.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: f.LocSuggestedToDeleteSum, + LocAddedSum: f.LocAddedSum, + LocDeletedSum: f.LocDeletedSum, + }, + }) + } + + // User by Language+Feature + for _, lf := range u.TotalsByLanguageFeature { + results = append(results, &models.GhCopilotUserMetricsByLanguageFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + Language: lf.Language, + Feature: lf.Feature, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lf.LocSuggestedToDeleteSum, + LocAddedSum: lf.LocAddedSum, + LocDeletedSum: lf.LocDeletedSum, + }, + }) + } + + // User by Language+Model + for _, lm := range u.TotalsByLanguageModel { + results = append(results, &models.GhCopilotUserMetricsByLanguageModel{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + Language: lm.Language, + Model: lm.Model, + CopilotCodeMetrics: models.CopilotCodeMetrics{ + CodeGenerationActivityCount: lm.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: lm.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: lm.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: lm.LocSuggestedToDeleteSum, + LocAddedSum: lm.LocAddedSum, + LocDeletedSum: lm.LocDeletedSum, + }, + }) + } + + // User by Model+Feature + for _, mf := range u.TotalsByModelFeature { + results = append(results, &models.GhCopilotUserMetricsByModelFeature{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + Model: mf.Model, + Feature: mf.Feature, + CopilotActivityMetrics: models.CopilotActivityMetrics{ + UserInitiatedInteractionCount: mf.UserInitiatedInteractionCount, + CodeGenerationActivityCount: mf.CodeGenerationActivityCount, + CodeAcceptanceActivityCount: mf.CodeAcceptanceActivityCount, + LocSuggestedToAddSum: mf.LocSuggestedToAddSum, + LocSuggestedToDeleteSum: mf.LocSuggestedToDeleteSum, + LocAddedSum: mf.LocAddedSum, + LocDeletedSum: mf.LocDeletedSum, + }, + }) + } + + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_teams_collector.go b/backend/plugins/gh-copilot/tasks/user_teams_collector.go new file mode 100644 index 00000000000..dc3287e5c5b --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_collector.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawUserTeamsTable = "copilot_user_teams" + +// CollectUserTeams collects user-team mapping data from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +func CollectUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + var urlTemplate string + + if connection.HasEnterprise() { + urlTemplate = copilotAPIPath("enterprises", connection.Enterprise, "copilot/metrics/reports/user-teams-1-day") + } else if connection.Organization != "" { + urlTemplate = copilotAPIPath("orgs", connection.Organization, "copilot/metrics/reports/user-teams-1-day") + } else { + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue + } + // User-teams reports are JSONL format + records, parseErr := parseJSONL(reportBody) + if parseErr != nil { + return nil, parseErr + } + results = append(results, records...) + } + return results, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_teams_extractor.go b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go new file mode 100644 index 00000000000..72a3de8abe9 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// userTeamRecord represents a single line from the user-teams-1-day JSONL report. +type userTeamRecord struct { + Day string `json:"day"` + UserId int64 `json:"user_id"` + UserLogin string `json:"user_login"` + OrganizationId string `json:"organization_id"` + EnterpriseId string `json:"enterprise_id"` + TeamId int64 `json:"team_id"` + Slug string `json:"slug"` +} + +// ExtractUserTeams parses user-team JSONL records into the GhCopilotUserTeam model. +func ExtractUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var rec userTeamRecord + if err := errors.Convert(json.Unmarshal(row.Data, &rec)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", rec.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in user-teams report") + } + + return []interface{}{ + &models.GhCopilotUserTeam{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: rec.UserId, + TeamId: rec.TeamId, + UserLogin: rec.UserLogin, + OrganizationId: rec.OrganizationId, + EnterpriseId: rec.EnterpriseId, + TeamSlug: rec.Slug, + }, + }, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/gitee/models/migrationscripts/archived/connection.go b/backend/plugins/gitee/models/migrationscripts/archived/connection.go index 55af7fc7713..14370ed586a 100644 --- a/backend/plugins/gitee/models/migrationscripts/archived/connection.go +++ b/backend/plugins/gitee/models/migrationscripts/archived/connection.go @@ -30,7 +30,7 @@ type RestConnection struct { BaseConnection `mapstructure:",squash"` Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` Proxy string `mapstructure:"proxy" json:"proxy"` - RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimit"` } type BaseConnection struct { diff --git a/backend/plugins/gitextractor/README.md b/backend/plugins/gitextractor/README.md index 623e5546620..f14178fcc63 100644 --- a/backend/plugins/gitextractor/README.md +++ b/backend/plugins/gitextractor/README.md @@ -14,4 +14,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/gitextractor) \ No newline at end of file +Please see details in the [Apache DevLake website](https://devlake.apache.org/docs/Plugins/gitextractor) + + +Build with local compiled v1.3.2 libgit2 +``` +CGO_LDFLAGS="-L/usr/local/lib -lgit2" CGO_CFLAGS="-I/usr/local/include" go build ./plugins/gitextractor/... 2>&1 | tail -5 +``` \ No newline at end of file diff --git a/backend/plugins/gitextractor/parser/clone_gitcli.go b/backend/plugins/gitextractor/parser/clone_gitcli.go index adeceb9e358..61931b521b5 100644 --- a/backend/plugins/gitextractor/parser/clone_gitcli.go +++ b/backend/plugins/gitextractor/parser/clone_gitcli.go @@ -113,37 +113,40 @@ func (g *GitcliCloner) prepareSync() errors.Error { } // support private key if taskData.Options.PrivateKey != "" { - pkFile, err := os.CreateTemp("", "gitext-pk") + // Create a restricted temp directory so the key file is protected from the moment of creation (CWE-367, CWE-732) + pkDir, err := os.MkdirTemp("", "gitext-pk-") if err != nil { - g.logger.Error(err, "create temp private key file error") + g.logger.Error(err, "create temp private key dir error") + return errors.Default.New("failed to handle the private key") + } + defer os.RemoveAll(pkDir) + pkFilePath := path.Join(pkDir, "pk") + pkFile, err := os.OpenFile(pkFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + g.logger.Error(err, "create private key file error") return errors.Default.New("failed to handle the private key") } if _, e := pkFile.WriteString(taskData.Options.PrivateKey + "\n"); e != nil { - g.logger.Error(err, "write private key file error") - return errors.Default.New("failed to write the private key") + g.logger.Error(e, "write private key file error") + return errors.Default.New("failed to write the private key") } pkFile.Close() - if e := os.Chmod(pkFile.Name(), 0600); e != nil { - g.logger.Error(err, "chmod private key file error") - return errors.Default.New("failed to modify the private key") - } if taskData.Options.Passphrase != "" { + // Pass passphrase via stdin to avoid exposure in /proc//cmdline (CWE-214) pp := exec.CommandContext( g.ctx.GetContext(), "ssh-keygen", "-p", - "-P", taskData.Options.Passphrase, "-N", "", - "-f", pkFile.Name(), + "-f", pkFilePath, ) - if ppout, pperr := pp.CombinedOutput(); pperr != nil { + pp.Stdin = strings.NewReader(taskData.Options.Passphrase + "\n") + if _, pperr := pp.CombinedOutput(); pperr != nil { g.logger.Error(pperr, "change private key passphrase error") - g.logger.Info(string(ppout)) return errors.Default.New("failed to decrypt the private key") } } - defer os.Remove(pkFile.Name()) - sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFile.Name())) + sshCmdArgs = append(sshCmdArgs, fmt.Sprintf("-i %s -o StrictHostKeyChecking=no", pkFilePath)) } if len(sshCmdArgs) > 0 { g.syncEnvs = append(g.syncEnvs, fmt.Sprintf("GIT_SSH_COMMAND=ssh %s", strings.Join(sshCmdArgs, " "))) @@ -251,6 +254,7 @@ func (g *GitcliCloner) doubleClone() errors.Error { if e != nil { return errors.Convert(e) } + defer os.RemoveAll(intermediaryDir) // CWE-459: ensure cleanup on all exit paths // step 1: full clone into a intermediary dir backup := g.localDir g.localDir = intermediaryDir @@ -303,7 +307,7 @@ func (g *GitcliCloner) gitCmd(gitcmd string, args ...string) errors.Error { } func (g *GitcliCloner) git(env []string, dir string, gitcmd string, args ...string) errors.Error { - g.logger.Debug("git %s %v", gitcmd, args) + g.logger.Debug("git %s %v", gitcmd, sanitizeArgs(args)) // CWE-532: sanitize before logging args = append([]string{gitcmd}, args...) cmd := exec.CommandContext(g.ctx.GetContext(), "git", args...) cmd.Env = env @@ -335,14 +339,47 @@ func generateErrMsg(output []byte, err error) string { return errMsg } +// sensitiveQueryParams lists URL query parameter names that may carry credentials. +var sensitiveQueryParams = []string{"token", "access_token", "private_token", "api_key", "key", "apikey"} + func sanitizeArgs(args []string) []string { var ret []string for _, arg := range args { + // Redact Authorization header values (e.g. --header=Authorization: Bearer TOKEN) (CWE-532) + lower := strings.ToLower(arg) + if authIdx := strings.Index(lower, "authorization:"); authIdx != -1 { + colonPos := authIdx + len("authorization:") + rest := strings.TrimSpace(arg[colonPos:]) + parts := strings.SplitN(rest, " ", 2) + if len(parts) == 2 { + arg = arg[:colonPos] + " " + parts[0] + " " + strings.Repeat("*", len(parts[1])) + } else if len(rest) > 0 { + arg = arg[:colonPos] + " " + strings.Repeat("*", len(rest)) + } + ret = append(ret, arg) + continue + } u, err := url.Parse(arg) if err == nil && u != nil && u.User != nil { password, ok := u.User.Password() if ok { + // Redact password in user:password@host URLs arg = strings.Replace(arg, password, strings.Repeat("*", len(password)), -1) + } else { + // Redact username-only tokens (e.g. https://TOKEN@github.com/..., GitHub App pattern) + username := u.User.Username() + if len(username) >= 16 { + arg = strings.Replace(arg, username, strings.Repeat("*", len(username)), -1) + } + } + } + // Redact sensitive query parameters (e.g. ?private_token=..., ?access_token=...) + if err == nil && u != nil && u.RawQuery != "" { + q := u.Query() + for _, param := range sensitiveQueryParams { + if val := q.Get(param); val != "" { + arg = strings.Replace(arg, val, strings.Repeat("*", len(val)), -1) + } } } ret = append(ret, arg) diff --git a/backend/plugins/gitextractor/parser/repo_gogit.go b/backend/plugins/gitextractor/parser/repo_gogit.go index 0e3d9f094d9..29deef315fb 100644 --- a/backend/plugins/gitextractor/parser/repo_gogit.go +++ b/backend/plugins/gitextractor/parser/repo_gogit.go @@ -400,7 +400,7 @@ func (r *GogitRepoCollector) storeParentCommits(commitSha string, commit *object for i := 0; i < commit.NumParents(); i++ { parent, err := commit.Parent(i) if err != nil { - // parent commit might not exist when repo is shallow cloned (tradeoff of supporting timeAfter paramenter) + // parent commit might not exist when repo is shallow cloned (tradeoff of supporting timeAfter parameter) if err.Error() == "object not found" { continue } diff --git a/backend/plugins/github/api/blueprint_v200.go b/backend/plugins/github/api/blueprint_v200.go index ffb464d5f75..a3646bd16ef 100644 --- a/backend/plugins/github/api/blueprint_v200.go +++ b/backend/plugins/github/api/blueprint_v200.go @@ -80,8 +80,11 @@ func makeDataSourcePipelinePlanV200( scopeDetails []*srvhelper.ScopeDetail[models.GithubRepo, models.GithubScopeConfig], connection *models.GithubConnection, ) (coreModels.PipelinePlan, errors.Error) { - plan := make(coreModels.PipelinePlan, len(scopeDetails)) + plan := make(coreModels.PipelinePlan, 0, len(scopeDetails)+1) for i, scopeDetail := range scopeDetails { + if i == len(plan) { + plan = append(plan, nil) + } githubRepo, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig stage := plan[i] if stage == nil { diff --git a/backend/plugins/github/api/remote_api.go b/backend/plugins/github/api/remote_api.go index 271fde3cf72..627f5b86df8 100644 --- a/backend/plugins/github/api/remote_api.go +++ b/backend/plugins/github/api/remote_api.go @@ -161,18 +161,18 @@ func listGithubOwnerRepos( ownerType, ownerID, err := getOwnerInfo(apiClient, owner) if err != nil { - return nil, nil, err + return } var reposBody *http.Response + var authUserID int switch ownerType { case "Organization": reposBody, err = apiClient.Get(fmt.Sprintf("orgs/%s/repos", owner), query, nil) case "User": - var authUserID int authUserID, err = getAuthenticatedUserID(apiClient) if err != nil { - return nil, nil, err + return } if authUserID == ownerID { // Authenticated user's own account - includes private repos @@ -186,7 +186,7 @@ func listGithubOwnerRepos( reposBody, err = apiClient.Get(fmt.Sprintf("users/%s/repos", owner), query, nil) } if err != nil { - return nil, nil, err + return } var repos []repo @@ -201,7 +201,7 @@ func listGithubOwnerRepos( PerPage: page.PerPage, } } - return children, nextPage, nil + return } func listGithubAppInstalledRepos( diff --git a/backend/plugins/github/e2e/account_test.go b/backend/plugins/github/e2e/account_test.go index 817ef6fab22..7b6466aba23 100644 --- a/backend/plugins/github/e2e/account_test.go +++ b/backend/plugins/github/e2e/account_test.go @@ -20,11 +20,15 @@ package e2e import ( "testing" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/github/impl" "github.com/apache/incubator-devlake/plugins/github/models" "github.com/apache/incubator-devlake/plugins/github/tasks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAccountDataFlow(t *testing.T) { @@ -108,4 +112,33 @@ func TestAccountDataFlow(t *testing.T) { "_raw_data_remark", }, ) + + // Referential-integrity invariant (#8886): every account the repo references in + // _tool_github_repo_accounts must have a domain `accounts` row, so issues.creator_id / + // pull_requests.author_id / merged_by_id never point at a missing account. We generate + // the domain id with the SAME generator the issue/PR convertors use, so this is a + // faithful proxy for the FK join the issue reported as broken. It also fails loudly if a + // future change shrinks ConvertAccounts' coverage or diverges the id generation. + accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) + var repoAccounts []models.GithubRepoAccount + require.NoError(t, dataflowTester.Dal.All(&repoAccounts, + dal.Where("repo_github_id = ? AND connection_id = ? AND account_id > 0", + taskData.Options.GithubId, taskData.Options.ConnectionId), + )) + require.NotEmpty(t, repoAccounts, "fixture must reference at least one account") + sawOrphanCase := false + for _, ra := range repoAccounts { + if ra.Login == "milichev" { + sawOrphanCase = true // the non-committer author from the issue repro + } + domainId := accountIdGen.Generate(taskData.Options.ConnectionId, ra.AccountId) + count, err := dataflowTester.Dal.Count( + dal.From(&crossdomain.Account{}), + dal.Where("id = ?", domainId), + ) + require.NoError(t, err) + assert.Equalf(t, int64(1), count, + "orphan FK: repo account %q (id=%d) has no domain accounts row %q", ra.Login, ra.AccountId, domainId) + } + assert.True(t, sawOrphanCase, "fixture should include the non-committer orphan case (milichev)") } diff --git a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv index 8b86664428f..7aa42a4b0a9 100644 --- a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv +++ b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv @@ -1,16 +1,17 @@ -connection_id,account_id,repo_github_id,login -1,21979,134018330,appleboy -1,964542,134018330,sarathsp06 -1,1052632,134018330,runner-mei -1,3794113,134018330,shanhuhai5739 -1,3971390,134018330,ppmoon -1,7496278,134018330,panjf2000 -1,8518239,134018330,gitter-badger -1,11763614,2,Moonlight-Zhao -1,12420699,2,shanghai-Jerry -1,14950473,2,zqkgo -1,22429695,2,codecov[bot] -1,24841832,2,rikewang -1,31087327,2,chensanle -1,32893410,2,zhangyuanxue -1,38849208,2,king526 \ No newline at end of file +connection_id,account_id,repo_github_id,login,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,21979,134018330,appleboy,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, +1,964542,134018330,sarathsp06,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,1, +1,1052632,134018330,runner-mei,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +1,3794113,134018330,shanhuhai5739,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, +1,3971390,134018330,ppmoon,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, +1,7496278,134018330,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,5, +1,8518239,134018330,gitter-badger,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,9, +1,145564,134018330,milichev,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, +1,11763614,2,Moonlight-Zhao,,,0, +1,12420699,2,shanghai-Jerry,,,0, +1,14950473,2,zqkgo,,,0, +1,22429695,2,codecov[bot],,,0, +1,24841832,2,rikewang,,,0, +1,31087327,2,chensanle,,,0, +1,32893410,2,zhangyuanxue,,,0, +1,38849208,2,king526,,,0, diff --git a/backend/plugins/github/e2e/snapshot_tables/account.csv b/backend/plugins/github/e2e/snapshot_tables/account.csv index 1092e7a5296..c3022a4a296 100644 --- a/backend/plugins/github/e2e/snapshot_tables/account.csv +++ b/backend/plugins/github/e2e/snapshot_tables/account.csv @@ -1,5 +1,6 @@ id,email,full_name,user_name,avatar_url,organization,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark github:GithubAccount:1:1052632,runner.mei@,runner,runner-mei,https://avatars.githubusercontent.com/u/1052632?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +github:GithubAccount:1:145564,,,milichev,,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, github:GithubAccount:1:21979,appleboy.tw@gmail.com,Bo-Yi Wu,appleboy,https://avatars.githubusercontent.com/u/21979?v=4,"COSCUP,nodejs-tw,moztw,h5bp,CodeIgniter-TW,drone,Getmore,golangtw,laravel-taiwan,go-xorm,gin-gonic,PHPConf-TW,Mediatek-Cloud,SJFinder,go-gitea,laradock,gin-contrib,tagfans,maintainers,go-training,go-ggz,the-benchmarker,golang-queue","{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, github:GithubAccount:1:3794113,shanhu5739@gmail.com,Derek,shanhuhai5739,https://avatars.githubusercontent.com/u/3794113?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, github:GithubAccount:1:3971390,cnliuyunpeng@gmail.com,ppmoon,ppmoon,https://avatars.githubusercontent.com/u/3971390?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, diff --git a/backend/plugins/github/models/connection.go b/backend/plugins/github/models/connection.go index 5897fe429fd..4489b70cb4a 100644 --- a/backend/plugins/github/models/connection.go +++ b/backend/plugins/github/models/connection.go @@ -62,11 +62,11 @@ type GithubConn struct { } // UpdateToken updates the token and refresh token information -func (conn *GithubConn) UpdateToken(newToken, newRefreshToken string, expiry, refreshExpiry time.Time) { +func (conn *GithubConn) UpdateToken(newToken, newRefreshToken string, expiry, refreshExpiry *time.Time) { conn.Token = newToken conn.RefreshToken = newRefreshToken - conn.TokenExpiresAt = &expiry - conn.RefreshTokenExpiresAt = &refreshExpiry + conn.TokenExpiresAt = expiry + conn.RefreshTokenExpiresAt = refreshExpiry // Update the internal tokens slice used by SetupAuthentication conn.tokens = []string{newToken} @@ -81,13 +81,15 @@ func (conn *GithubConn) PrepareApiClient(apiClient plugin.ApiClient) errors.Erro } if conn.AuthMethod == AppKey && conn.InstallationID != 0 { - token, err := conn.getInstallationAccessToken(apiClient) + token, err := conn.GetInstallationAccessToken(apiClient) if err != nil { return err } - - conn.Token = token.Token - conn.tokens = []string{token.Token} + var expiresAt *time.Time + if !token.ExpiresAt.IsZero() { + expiresAt = &token.ExpiresAt + } + conn.UpdateToken(token.Token, "", expiresAt, nil) } return nil @@ -364,7 +366,8 @@ type GithubUserOfToken struct { } type InstallationToken struct { - Token string `json:"token"` + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` } type GithubApp struct { @@ -407,7 +410,7 @@ func (gak *GithubAppKey) CreateJwt() (string, errors.Error) { return tokenString, nil } -func (gak *GithubAppKey) getInstallationAccessToken( +func (gak *GithubAppKey) GetInstallationAccessToken( apiClient plugin.ApiClient, ) (*InstallationToken, errors.Error) { @@ -427,12 +430,18 @@ func (gak *GithubAppKey) getInstallationAccessToken( if err != nil { return nil, err } + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(resp.StatusCode).New(fmt.Sprintf("unexpected status code while getting installation access token: %s", string(body))) + } var installationToken InstallationToken err = errors.Convert(json.Unmarshal(body, &installationToken)) if err != nil { return nil, err } + if installationToken.Token == "" { + return nil, errors.Default.New("empty installation access token returned") + } return &installationToken, nil } diff --git a/backend/plugins/github/models/migrationscripts/20260211_modify_connection_token_expires_at_to_nullable.go b/backend/plugins/github/models/migrationscripts/20260211_modify_connection_token_expires_at_to_nullable.go new file mode 100644 index 00000000000..ff7b127fb09 --- /dev/null +++ b/backend/plugins/github/models/migrationscripts/20260211_modify_connection_token_expires_at_to_nullable.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type githubConnection20260211 struct { + TokenExpiresAt *time.Time + RefreshTokenExpiresAt *time.Time +} + +func (githubConnection20260211) TableName() string { + return "_tool_github_connections" +} + +type modifyTokenExpiresAtToNullable struct{} + +func (*modifyTokenExpiresAtToNullable) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &githubConnection20260211{}, + ) +} + +func (*modifyTokenExpiresAtToNullable) Version() uint64 { + return 20260211000001 +} + +func (*modifyTokenExpiresAtToNullable) Name() string { + return "modify token_expires_at and refresh_token_expires_at to nullable" +} diff --git a/backend/plugins/github/models/migrationscripts/register.go b/backend/plugins/github/models/migrationscripts/register.go index 8c8e24c47fc..c89937f94c8 100644 --- a/backend/plugins/github/models/migrationscripts/register.go +++ b/backend/plugins/github/models/migrationscripts/register.go @@ -57,6 +57,7 @@ func All() []plugin.MigrationScript { new(addIndexToGithubJobs), new(addRefreshTokenFields), new(fixNullableTokenExpiry), + new(modifyTokenExpiresAtToNullable), new(addPrSizeExcludedFileExtensions), } } diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go index dfd9a929ee9..3d4db97d155 100644 --- a/backend/plugins/github/tasks/account_convertor.go +++ b/backend/plugins/github/tasks/account_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "strings" "github.com/apache/incubator-devlake/core/dal" @@ -32,6 +31,19 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +// repoAccountForConvert is the row projected by ConvertAccounts' query: every +// account referenced by the repo (from _tool_github_repo_accounts), enriched +// with profile detail from _tool_github_accounts when it was collected. The +// embedded NoPKModel carries the RawDataOrigin across to the domain row. +type repoAccountForConvert struct { + Id int + Login string + Name string + Email string + AvatarUrl string + common.NoPKModel +} + func init() { RegisterSubtaskMeta(&ConvertAccountsMeta) } @@ -40,59 +52,81 @@ var ConvertAccountsMeta = plugin.SubTaskMeta{ Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, - Description: "Convert tool layer table github_accounts into domain layer table accounts", + Description: "Convert every account referenced by the repo (tool layer repo_accounts, enriched by github_accounts) into domain layer table accounts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, DependencyTables: []string{ - models.GithubAccount{}.TableName(), // cursor - models.GithubRepoAccount{}.TableName(), // cursor - models.GithubAccountOrg{}.TableName()}, // account id gen + models.GithubRepoAccount{}.TableName(), // cursor (every user referenced by the repo) + models.GithubAccount{}.TableName(), // left-join enrichment (profile detail, optional) + models.GithubAccountOrg{}.TableName()}, // org pluck ProductTables: []string{crossdomain.Account{}.TableName()}, } -type GithubAccountWithOrg struct { - models.GithubAccount - Login string `json:"login" gorm:"type:varchar(255)"` - common.NoPKModel -} - func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*GithubTaskData) - cursor, err := db.Cursor( - dal.Select("_tool_github_accounts.*"), - dal.From(&models.GithubAccount{}), - dal.Where( - "repo_github_id = ? and _tool_github_accounts.connection_id=?", - data.Options.GithubId, - data.Options.ConnectionId, - ), - dal.Join(`left join _tool_github_repo_accounts gra on ( - _tool_github_accounts.connection_id = gra.connection_id - AND _tool_github_accounts.id = gra.account_id - )`), - ) - if err != nil { - return err - } - defer cursor.Close() - accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubAccount{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[repoAccountForConvert]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_ACCOUNT_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_ACCOUNT_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubUser := inputRow.(*models.GithubAccount) - + // Source every account referenced by this repo from _tool_github_repo_accounts + // (which the issue/PR/commit extractors populate for any author, assignee, or + // merged-by user), and LEFT JOIN _tool_github_accounts for profile detail when it + // was collected. This guarantees a domain `accounts` row for every CreatorId / + // AuthorId the other convertors emit, instead of only for users who committed. + // Raw-data provenance follows the same rule as the profile fields: the enriched + // _tool_github_accounts row when we collected one, the repo_accounts row otherwise. + // Note the consequence: fallback-provenance rows carry a _raw_data_table other than + // _raw_github_api_accounts, so the batch-save divider's full-sync delete-then-reinsert + // (keyed on this converter's raw table) never deletes them; they are reconciled by + // upsert only. Scope deletion still covers them via _raw_data_params. + // SQL is kept DB-agnostic (no backtick quoting, COALESCE not IFNULL) so it runs on + // both MySQL and PostgreSQL. + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select(`_tool_github_repo_accounts.account_id AS id, + _tool_github_repo_accounts.login AS login, + COALESCE(ga.name, '') AS name, + COALESCE(ga.email, '') AS email, + COALESCE(ga.avatar_url, '') AS avatar_url, + COALESCE(ga._raw_data_params, _tool_github_repo_accounts._raw_data_params) AS _raw_data_params, + COALESCE(ga._raw_data_table, _tool_github_repo_accounts._raw_data_table) AS _raw_data_table, + COALESCE(ga._raw_data_id, _tool_github_repo_accounts._raw_data_id) AS _raw_data_id, + COALESCE(ga._raw_data_remark, _tool_github_repo_accounts._raw_data_remark) AS _raw_data_remark`), + dal.From(&models.GithubRepoAccount{}), + dal.Join(`left join _tool_github_accounts ga on ( + ga.connection_id = _tool_github_repo_accounts.connection_id + AND ga.id = _tool_github_repo_accounts.account_id + )`), + dal.Where( + `_tool_github_repo_accounts.repo_github_id = ? + AND _tool_github_repo_accounts.connection_id = ? + AND _tool_github_repo_accounts.account_id > 0`, + data.Options.GithubId, + data.Options.ConnectionId, + ), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + // Incremental cursor intentionally tracks _tool_github_repo_accounts.updated_at + // (repo membership), not _tool_github_accounts.updated_at (profile freshness): + // account-detail re-enrichment is reconciled on the next full sync. Do not switch + // this back to _tool_github_accounts — that is what left issue/PR-only authors + // orphaned (#8886). + clauses = append(clauses, dal.Where("_tool_github_repo_accounts.updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubUser *repoAccountForConvert) ([]interface{}, errors.Error) { // query related orgs var orgs []string err := db.Pluck(`org_login`, &orgs, diff --git a/backend/plugins/github/tasks/account_extractor.go b/backend/plugins/github/tasks/account_extractor.go index 88e4e8efb48..27500402b8a 100644 --- a/backend/plugins/github/tasks/account_extractor.go +++ b/backend/plugins/github/tasks/account_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "time" "github.com/apache/incubator-devlake/core/errors" @@ -61,39 +60,32 @@ type DetailGithubAccountResponse struct { func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[DetailGithubAccountResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_ACCOUNT_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - apiAccount := &DetailGithubAccountResponse{} - err := errors.Convert(json.Unmarshal(row.Data, apiAccount)) - if err != nil { - return nil, err - } - results := make([]interface{}, 0, 1) - if apiAccount.Id == 0 { + Extract: func(body *DetailGithubAccountResponse, row *api.RawData) ([]any, errors.Error) { + if body.Id == 0 { return nil, nil } githubAccount := &models.GithubAccount{ ConnectionId: data.Options.ConnectionId, - Id: apiAccount.Id, - Login: apiAccount.Login, - Name: apiAccount.Name, - Company: apiAccount.Company, - Email: apiAccount.Email, - AvatarUrl: apiAccount.AvatarUrl, - Url: apiAccount.Url, - HtmlUrl: apiAccount.HtmlUrl, - Type: apiAccount.Type, + Id: body.Id, + Login: body.Login, + Name: body.Name, + Company: body.Company, + Email: body.Email, + AvatarUrl: body.AvatarUrl, + Url: body.Url, + HtmlUrl: body.HtmlUrl, + Type: body.Type, } - results = append(results, githubAccount) - return results, nil + return []any{githubAccount}, nil }, }) diff --git a/backend/plugins/github/tasks/account_org_extractor.go b/backend/plugins/github/tasks/account_org_extractor.go index 8aa69b664c7..91f9a7b8cfe 100644 --- a/backend/plugins/github/tasks/account_org_extractor.go +++ b/backend/plugins/github/tasks/account_org_extractor.go @@ -50,28 +50,23 @@ type GithubAccountOrgsResponse struct { func ExtractAccountOrg(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[[]GithubAccountOrgsResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_ACCOUNT_ORG_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - apiAccountOrgs := &[]GithubAccountOrgsResponse{} - err := json.Unmarshal(row.Data, apiAccountOrgs) - if err != nil { - return nil, errors.Convert(err) - } + Extract: func(body *[]GithubAccountOrgsResponse, row *api.RawData) ([]any, errors.Error) { simpleAccount := &SimpleAccountWithId{} - err = json.Unmarshal(row.Input, simpleAccount) + err := errors.Convert(json.Unmarshal(row.Input, simpleAccount)) if err != nil { - return nil, errors.Convert(err) + return nil, err } - results := make([]interface{}, 0, len(*apiAccountOrgs)) - for _, apiAccountOrg := range *apiAccountOrgs { + results := make([]interface{}, 0, len(*body)) + for _, apiAccountOrg := range *body { githubAccount := &models.GithubAccountOrg{ ConnectionId: data.Options.ConnectionId, AccountId: simpleAccount.AccountId, diff --git a/backend/plugins/github/tasks/api_client.go b/backend/plugins/github/tasks/api_client.go index 268af8ecec1..c5be8a0f22b 100644 --- a/backend/plugins/github/tasks/api_client.go +++ b/backend/plugins/github/tasks/api_client.go @@ -26,7 +26,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "github.com/apache/incubator-devlake/plugins/github/token" ) func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnection) (*api.ApiAsyncClient, errors.Error) { @@ -35,22 +34,10 @@ func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnec return nil, err } - // Inject TokenProvider if refresh token is present - if connection.RefreshToken != "" { - logger := taskCtx.GetLogger() - db := taskCtx.GetDal() - - // Create TokenProvider - tp := token.NewTokenProvider(connection, db, apiClient.GetClient(), logger) - - // Wrap the transport - baseTransport := apiClient.GetClient().Transport - if baseTransport == nil { - baseTransport = http.DefaultTransport - } - - rt := token.NewRefreshRoundTripper(baseTransport, tp) - apiClient.GetClient().Transport = rt + // inject the shared auth layer + _, err = CreateAuthenticatedHttpClient(taskCtx, connection, apiClient.GetClient()) + if err != nil { + return nil, err } // create rate limit calculator diff --git a/backend/plugins/github/tasks/cicd_job_convertor.go b/backend/plugins/github/tasks/cicd_job_convertor.go index 81ef9cd4df5..2dcfbce37fc 100644 --- a/backend/plugins/github/tasks/cicd_job_convertor.go +++ b/backend/plugins/github/tasks/cicd_job_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -55,37 +53,38 @@ type SimpleBranch struct { func ConvertJobs(taskCtx plugin.SubTaskContext) (err errors.Error) { db := taskCtx.GetDal() + logger := taskCtx.GetLogger() data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - if err != nil { - return err - } - job := &models.GithubJob{} - cursor, err := db.Cursor( - dal.From(job), - dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() + jobIdGen := didgen.NewDomainIdGenerator(&models.GithubJob{}) runIdGen := didgen.NewDomainIdGenerator(&models.GithubRun{}) repoIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubJob]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_JOB_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_JOB_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubJob{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - line := inputRow.(*models.GithubJob) - + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubJob{}), + dal.Where("_tool_github_jobs.repo_id = ? and _tool_github_jobs.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + logger.Info("[Convert Jobs] incremental mode using _tool_github_jobs.updated_at >= %v", since) + clauses = append(clauses, dal.Where("_tool_github_jobs.updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(line *models.GithubJob) ([]interface{}, errors.Error) { // Skip jobs with no started_at value (workaround for https://github.com/apache/incubator-devlake/issues/8442) if line.StartedAt == nil { return nil, nil diff --git a/backend/plugins/github/tasks/cicd_job_extractor.go b/backend/plugins/github/tasks/cicd_job_extractor.go index 5701476bcce..6cfab02f3cf 100644 --- a/backend/plugins/github/tasks/cicd_job_extractor.go +++ b/backend/plugins/github/tasks/cicd_job_extractor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" "strings" "github.com/apache/incubator-devlake/core/errors" @@ -46,23 +45,16 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[models.GithubJob]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_JOB_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - githubJob := &models.GithubJob{} - err := errors.Convert(json.Unmarshal(row.Data, githubJob)) - if err != nil { - return nil, err - } - - results := make([]interface{}, 0, 1) + Extract: func(githubJob *models.GithubJob, row *api.RawData) ([]any, errors.Error) { githubJobResult := &models.GithubJob{ ConnectionId: data.Options.ConnectionId, RepoId: repoId, @@ -87,8 +79,7 @@ func ExtractJobs(taskCtx plugin.SubTaskContext) errors.Error { Type: data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, githubJob.Name), Environment: data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, githubJob.Name), } - results = append(results, githubJobResult) - return results, nil + return []any{githubJobResult}, nil }, }) diff --git a/backend/plugins/github/tasks/cicd_run_collector.go b/backend/plugins/github/tasks/cicd_run_collector.go index 44c816ca236..9359b98de61 100644 --- a/backend/plugins/github/tasks/cicd_run_collector.go +++ b/backend/plugins/github/tasks/cicd_run_collector.go @@ -24,10 +24,12 @@ import ( "net/url" "time" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" ) func init() { @@ -89,6 +91,24 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { // `windowStart` past the previously collected second (inclusive-both-ends), while // fullsync + TimeAfter keeps the user-specified bound inclusive. createdAfter := manager.GetSince() + sinceSource := "state_since" + syncPolicy := taskCtx.TaskContext().SyncPolicy() + if createdAfter == nil { + sinceSource = "none" + } + if createdAfter == nil && (syncPolicy == nil || !syncPolicy.FullSync) { + fallbackSince, err := loadLatestRunUpdatedAt(taskCtx, data.Options.ConnectionId, data.Options.GithubId) + if err != nil { + return err + } + if fallbackSince != nil { + createdAfter = fallbackSince + sinceSource = "tool_runs_fallback" + logger.Info("cicd_run_collector: collector state missing; bootstrapping since from existing _tool_github_runs at %s", fallbackSince.UTC().Format(time.RFC3339)) + } else { + logger.Debug("cicd_run_collector: collector state missing and no _tool_github_runs timestamp found") + } + } untilPtr := manager.GetUntil() *untilPtr = untilPtr.Truncate(time.Second) until := *untilPtr @@ -102,12 +122,14 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { } else { // 2018-01-01 conservatively predates GitHub Actions' late-2019 GA. windowStart = time.Date(2018, 1, 1, 0, 0, 0, 0, time.UTC) + sinceSource = "epoch_fullsync" } - logger.Info("cicd_run_collector: collecting workflow runs in [%s, %s] (incremental=%v)", + logger.Info("cicd_run_collector: collecting workflow runs in [%s, %s] (incremental=%v, since_source=%s)", windowStart.Format(githubTimeLayout), until.Format(githubTimeLayout), - manager.IsIncremental()) + manager.IsIncremental(), + sinceSource) leafWindows, err := newLeafWindowBuilder(taskCtx, data).build(windowStart, until) if err != nil { @@ -122,6 +144,28 @@ func CollectRuns(taskCtx plugin.SubTaskContext) errors.Error { return manager.Execute() } +func loadLatestRunUpdatedAt(taskCtx plugin.SubTaskContext, connectionId uint64, repoId int) (*time.Time, errors.Error) { + db := taskCtx.GetDal() + latest := &models.GithubRun{} + err := db.First( + latest, + dal.Where("connection_id = ? AND repo_id = ? AND github_updated_at IS NOT NULL", connectionId, repoId), + dal.Orderby("github_updated_at DESC"), + dal.Limit(1), + ) + if err != nil { + if db.IsErrorNotFound(err) { + return nil, nil + } + return nil, err + } + if latest.GithubUpdatedAt == nil { + return nil, nil + } + fallback := latest.GithubUpdatedAt.UTC() + return &fallback, nil +} + // buildRunsQuery assembles the filtered-mode query for a single leaf TimeWindow. // Shared between registerCollectorForLeafWindows and tests. func buildRunsQuery(reqData *helper.RequestData) (url.Values, errors.Error) { diff --git a/backend/plugins/github/tasks/cicd_run_collector_test.go b/backend/plugins/github/tasks/cicd_run_collector_test.go index d69b5c73b59..a7a12abe6b9 100644 --- a/backend/plugins/github/tasks/cicd_run_collector_test.go +++ b/backend/plugins/github/tasks/cicd_run_collector_test.go @@ -34,12 +34,64 @@ import ( "github.com/apache/incubator-devlake/helpers/unithelper" mockdal "github.com/apache/incubator-devlake/mocks/core/dal" mockapi "github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) +func TestCicdRunLoadLatestRunUpdatedAt_ReturnsLatestTimestamp(t *testing.T) { + connectionId := uint64(1) + repoId := 101 + latestTs := time.Date(2025, 4, 1, 10, 11, 12, 0, time.UTC) + + mockDal := new(mockdal.Dal) + mockDal.On("First", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + dst := args.Get(0).(*models.GithubRun) + dst.ConnectionId = connectionId + dst.RepoId = repoId + dst.GithubUpdatedAt = &latestTs + }).Return(nil).Once() + + ctx := unithelper.DummySubTaskContext(mockDal) + since, err := loadLatestRunUpdatedAt(ctx, connectionId, repoId) + + require.Nil(t, err) + require.NotNil(t, since) + assert.True(t, since.Equal(latestTs)) + mockDal.AssertExpectations(t) +} + +func TestCicdRunLoadLatestRunUpdatedAt_NotFoundReturnsNil(t *testing.T) { + mockDal := new(mockdal.Dal) + notFoundErr := errors.Default.New("record not found") + mockDal.On("First", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(notFoundErr).Once() + mockDal.On("IsErrorNotFound", notFoundErr).Return(true).Once() + + ctx := unithelper.DummySubTaskContext(mockDal) + since, err := loadLatestRunUpdatedAt(ctx, 1, 101) + + require.Nil(t, err) + assert.Nil(t, since) + mockDal.AssertExpectations(t) +} + +func TestCicdRunLoadLatestRunUpdatedAt_PropagatesError(t *testing.T) { + mockDal := new(mockdal.Dal) + dbErr := errors.Default.New("db unavailable") + mockDal.On("First", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(dbErr).Once() + mockDal.On("IsErrorNotFound", dbErr).Return(false).Once() + + ctx := unithelper.DummySubTaskContext(mockDal) + since, err := loadLatestRunUpdatedAt(ctx, 1, 101) + + assert.Nil(t, since) + require.NotNil(t, err) + assert.Contains(t, err.Error(), "db unavailable") + mockDal.AssertExpectations(t) +} + // newTestBuilder constructs a leafWindowBuilder with a stubbed probe for unit testing. func newTestBuilder(probe probeFunc) *leafWindowBuilder { mockDal := new(mockdal.Dal) diff --git a/backend/plugins/github/tasks/cicd_run_convertor.go b/backend/plugins/github/tasks/cicd_run_convertor.go index 1cdf9c6daa5..69911482e47 100644 --- a/backend/plugins/github/tasks/cicd_run_convertor.go +++ b/backend/plugins/github/tasks/cicd_run_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "time" "github.com/apache/incubator-devlake/core/dal" @@ -63,31 +62,33 @@ func ConvertRuns(taskCtx plugin.SubTaskContext) errors.Error { return err } - pipeline := &models.GithubRun{} - cursor, err := db.Cursor( - dal.Select("*"), - dal.From(pipeline), - dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() repoIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) runIdGen := didgen.NewDomainIdGenerator(&models.GithubRun{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubRun]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_RUN_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_RUN_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubRun{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - line := inputRow.(*models.GithubRun) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.GithubRun{}), + dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(line *models.GithubRun) ([]interface{}, errors.Error) { createdAt := time.Now() if line.GithubCreatedAt != nil { createdAt = *line.GithubCreatedAt diff --git a/backend/plugins/github/tasks/cicd_run_extractor.go b/backend/plugins/github/tasks/cicd_run_extractor.go index 1e6a3f41a9e..3a474c64c02 100644 --- a/backend/plugins/github/tasks/cicd_run_extractor.go +++ b/backend/plugins/github/tasks/cicd_run_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" @@ -45,26 +43,21 @@ func ExtractRuns(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[models.GithubRun]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_RUN_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - githubRun := &models.GithubRun{} - err := errors.Convert(json.Unmarshal(row.Data, githubRun)) - if err != nil { - return nil, err - } - githubRun.RepoId = repoId - githubRun.ConnectionId = data.Options.ConnectionId - githubRun.Type = data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, githubRun.Name) - githubRun.Environment = data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, githubRun.Name, githubRun.HeadBranch) - return []interface{}{githubRun}, nil + Extract: func(body *models.GithubRun, row *api.RawData) ([]any, errors.Error) { + body.RepoId = repoId + body.ConnectionId = data.Options.ConnectionId + body.Type = data.RegexEnricher.ReturnNameIfMatched(devops.DEPLOYMENT, body.Name) + body.Environment = data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION, body.Name, body.HeadBranch) + return []any{body}, nil }, }) diff --git a/backend/plugins/github/tasks/comment_collector.go b/backend/plugins/github/tasks/comment_collector.go index 9f40e7736a7..303ecf7b1f7 100644 --- a/backend/plugins/github/tasks/comment_collector.go +++ b/backend/plugins/github/tasks/comment_collector.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "net/url" + "time" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -66,7 +67,7 @@ func CollectApiComments(taskCtx plugin.SubTaskContext) errors.Error { query := url.Values{} query.Set("state", "all") if apiCollector.GetSince() != nil { - query.Set("since", apiCollector.GetSince().String()) + query.Set("since", apiCollector.GetSince().UTC().Format(time.RFC3339)) } query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("direction", "asc") diff --git a/backend/plugins/github/tasks/comment_extractor.go b/backend/plugins/github/tasks/comment_extractor.go index 8fc52a18b7e..c383e532e9c 100644 --- a/backend/plugins/github/tasks/comment_extractor.go +++ b/backend/plugins/github/tasks/comment_extractor.go @@ -59,28 +59,21 @@ type IssueComment struct { func ExtractApiComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[IssueComment]{ + SubtaskCommonArgs: &helper.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_COMMENTS_TABLE, }, - Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - apiComment := &IssueComment{} - err := errors.Convert(json.Unmarshal(row.Data, apiComment)) - if err != nil { - return nil, err - } - // need to extract 2 kinds of entities here + Extract: func(body *IssueComment, row *helper.RawData) ([]any, errors.Error) { results := make([]interface{}, 0, 2) - if apiComment.GithubId == 0 { + if body.GithubId == 0 { return nil, nil } - //If this is a pr, ignore - issueINumber, err := errors.Convert01(githubUtils.GetIssueIdByIssueUrl(apiComment.IssueUrl)) + issueINumber, err := errors.Convert01(githubUtils.GetIssueIdByIssueUrl(body.IssueUrl)) if err != nil { return nil, err } @@ -90,7 +83,7 @@ func ExtractApiComments(taskCtx plugin.SubTaskContext) errors.Error { if err != nil && !db.IsErrorNotFound(err) { return nil, err } - //if we can not find issues with issue number above, move the comments to github_pull_request_comments + // if we can not find issues with issue number above, move the comments to github_pull_request_comments if issue.GithubId == 0 { pr := &models.GithubPullRequest{} err = db.First(pr, dal.Where("connection_id = ? and number = ? and repo_id = ?", data.Options.ConnectionId, issueINumber, data.Options.GithubId)) @@ -99,23 +92,21 @@ func ExtractApiComments(taskCtx plugin.SubTaskContext) errors.Error { } githubPrComment := &models.GithubPrComment{ ConnectionId: data.Options.ConnectionId, - GithubId: apiComment.GithubId, + GithubId: body.GithubId, PullRequestId: pr.GithubId, - Body: string(apiComment.Body), - GithubCreatedAt: apiComment.GithubCreatedAt.ToTime(), - GithubUpdatedAt: apiComment.GithubUpdatedAt.ToTime(), + Body: string(body.Body), + GithubCreatedAt: body.GithubCreatedAt.ToTime(), + GithubUpdatedAt: body.GithubUpdatedAt.ToTime(), Type: "NORMAL", } - if apiComment.User != nil { - // Filter bot comments by username - if shouldSkipByUsername(apiComment.User.Login) { - taskCtx.GetLogger().Debug("Skipping PR comment #%d from bot user: %s", apiComment.GithubId, apiComment.User.Login) + if body.User != nil { + if shouldSkipByUsername(body.User.Login) { + taskCtx.GetLogger().Debug("Skipping PR comment #%d from bot user: %s", body.GithubId, body.User.Login) return nil, nil } - githubPrComment.AuthorUsername = apiComment.User.Login - githubPrComment.AuthorUserId = apiComment.User.Id - - githubAccount, err := convertAccount(apiComment.User, data.Options.GithubId, data.Options.ConnectionId) + githubPrComment.AuthorUsername = body.User.Login + githubPrComment.AuthorUserId = body.User.Id + githubAccount, err := convertAccount(body.User, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } @@ -125,22 +116,20 @@ func ExtractApiComments(taskCtx plugin.SubTaskContext) errors.Error { } else { githubIssueComment := &models.GithubIssueComment{ ConnectionId: data.Options.ConnectionId, - GithubId: apiComment.GithubId, + GithubId: body.GithubId, IssueId: issue.GithubId, - Body: string(apiComment.Body), - GithubCreatedAt: apiComment.GithubCreatedAt.ToTime(), - GithubUpdatedAt: apiComment.GithubUpdatedAt.ToTime(), + Body: string(body.Body), + GithubCreatedAt: body.GithubCreatedAt.ToTime(), + GithubUpdatedAt: body.GithubUpdatedAt.ToTime(), } - if apiComment.User != nil { - // Filter bot comments by username - if shouldSkipByUsername(apiComment.User.Login) { - taskCtx.GetLogger().Debug("Skipping issue comment #%d from bot user: %s", apiComment.GithubId, apiComment.User.Login) + if body.User != nil { + if shouldSkipByUsername(body.User.Login) { + taskCtx.GetLogger().Debug("Skipping issue comment #%d from bot user: %s", body.GithubId, body.User.Login) return nil, nil } - githubIssueComment.AuthorUsername = apiComment.User.Login - githubIssueComment.AuthorUserId = apiComment.User.Id - - githubAccount, err := convertAccount(apiComment.User, data.Options.GithubId, data.Options.ConnectionId) + githubIssueComment.AuthorUsername = body.User.Login + githubIssueComment.AuthorUserId = body.User.Id + githubAccount, err := convertAccount(body.User, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } diff --git a/backend/plugins/github/tasks/commit_collector.go b/backend/plugins/github/tasks/commit_collector.go index 019060c17d8..3376f248665 100644 --- a/backend/plugins/github/tasks/commit_collector.go +++ b/backend/plugins/github/tasks/commit_collector.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "net/url" + "time" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -78,7 +79,7 @@ func CollectApiCommits(taskCtx plugin.SubTaskContext) errors.Error { query := url.Values{} query.Set("state", "all") if apiCollector.GetSince() != nil { - query.Set("since", apiCollector.GetSince().String()) + query.Set("since", apiCollector.GetSince().UTC().Format(time.RFC3339)) } query.Set("direction", "asc") query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) diff --git a/backend/plugins/github/tasks/commit_convertor.go b/backend/plugins/github/tasks/commit_convertor.go index 4dc3fa43cc0..14448d48b1c 100644 --- a/backend/plugins/github/tasks/commit_convertor.go +++ b/backend/plugins/github/tasks/commit_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -54,36 +52,36 @@ func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From("_tool_github_commits gc"), - dal.Join(`left join _tool_github_repo_commits grc on ( - grc.commit_sha = gc.sha - )`), - dal.Select("gc.*"), - dal.Where("grc.repo_id = ? AND grc.connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - repoDidGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) domainRepoId := repoDidGen.Generate(data.Options.ConnectionId, repoId) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubCommit]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_COMMIT_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_COMMIT_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubCommit{}), - Input: cursor, - - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubCommit := inputRow.(*models.GithubCommit) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From("_tool_github_commits gc"), + dal.Join(`left join _tool_github_repo_commits grc on ( + grc.commit_sha = gc.sha + )`), + dal.Select("gc.*"), + dal.Where("grc.repo_id = ? AND grc.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("(gc.updated_at >= ? OR grc.updated_at >= ?)", since, since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubCommit *models.GithubCommit) ([]interface{}, errors.Error) { domainCommit := &code.Commit{ Sha: githubCommit.Sha, Message: githubCommit.Message, diff --git a/backend/plugins/github/tasks/commit_extractor.go b/backend/plugins/github/tasks/commit_extractor.go index a0ed8d11d3f..72a29454dac 100644 --- a/backend/plugins/github/tasks/commit_extractor.go +++ b/backend/plugins/github/tasks/commit_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -68,60 +66,44 @@ type Commit struct { func ExtractApiCommits(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraCommits by Board - */ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[CommitsResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_COMMIT_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - commit := &CommitsResponse{} - err := errors.Convert(json.Unmarshal(row.Data, commit)) - if err != nil { - return nil, err - } - if commit.Sha == "" { + Extract: func(body *CommitsResponse, row *api.RawData) ([]any, errors.Error) { + if body.Sha == "" { return nil, nil } - results := make([]interface{}, 0, 4) - githubCommit := &models.GithubCommit{ - Sha: commit.Sha, - Message: commit.Commit.Message, - AuthorName: commit.Commit.Author.Name, - AuthorEmail: commit.Commit.Author.Email, - AuthoredDate: commit.Commit.Author.Date.ToTime(), - CommitterName: commit.Commit.Committer.Name, - CommitterEmail: commit.Commit.Committer.Email, - CommittedDate: commit.Commit.Committer.Date.ToTime(), - Url: commit.Url, + Sha: body.Sha, + Message: body.Commit.Message, + AuthorName: body.Commit.Author.Name, + AuthorEmail: body.Commit.Author.Email, + AuthoredDate: body.Commit.Author.Date.ToTime(), + CommitterName: body.Commit.Committer.Name, + CommitterEmail: body.Commit.Committer.Email, + CommittedDate: body.Commit.Committer.Date.ToTime(), + Url: body.Url, } - if commit.Author != nil { - githubCommit.AuthorId = commit.Author.Id - results = append(results, commit.Author) + if body.Author != nil { + githubCommit.AuthorId = body.Author.Id + results = append(results, body.Author) } - if commit.Committer != nil { - githubCommit.CommitterId = commit.Committer.Id - results = append(results, commit.Committer) + if body.Committer != nil { + githubCommit.CommitterId = body.Committer.Id + results = append(results, body.Committer) } - githubRepoCommit := &models.GithubRepoCommit{ ConnectionId: data.Options.ConnectionId, RepoId: data.Options.GithubId, - CommitSha: commit.Sha, + CommitSha: body.Sha, } - results = append(results, githubCommit) results = append(results, githubRepoCommit) return results, nil diff --git a/backend/plugins/github/tasks/commit_stats_extractor.go b/backend/plugins/github/tasks/commit_stats_extractor.go index 5f99b1ed5bc..431f2d28ea1 100644 --- a/backend/plugins/github/tasks/commit_stats_extractor.go +++ b/backend/plugins/github/tasks/commit_stats_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -62,42 +60,27 @@ type ApiSingleCommitResponse struct { func ExtractApiCommitStats(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraCommits by Board - */ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[ApiSingleCommitResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_COMMIT_STATS_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - body := &ApiSingleCommitResponse{} - err := errors.Convert(json.Unmarshal(row.Data, body)) - if err != nil { - return nil, err - } + Extract: func(body *ApiSingleCommitResponse, row *api.RawData) ([]any, errors.Error) { if body.Sha == "" { return nil, nil } - db := taskCtx.GetDal() commit := &models.GithubCommit{} - err = db.First(commit, dal.Where("sha = ?", body.Sha), dal.Limit(1)) + err := db.First(commit, dal.Where("sha = ?", body.Sha), dal.Limit(1)) if err != nil && !db.IsErrorNotFound(err) { return nil, err } - commit.Additions = body.Stats.Additions commit.Deletions = body.Stats.Deletions - commitStat := &models.GithubCommitStat{ ConnectionId: data.Options.ConnectionId, Additions: body.Stats.Additions, @@ -105,13 +88,7 @@ func ExtractApiCommitStats(taskCtx plugin.SubTaskContext) errors.Error { CommittedDate: body.Commit.Committer.Date.ToTime(), Sha: body.Sha, } - - results := make([]interface{}, 0, 2) - - results = append(results, commit) - results = append(results, commitStat) - - return results, nil + return []any{commit, commitStat}, nil }, }) diff --git a/backend/plugins/github/tasks/event_extractor.go b/backend/plugins/github/tasks/event_extractor.go index 133527d168d..01b5495be3e 100644 --- a/backend/plugins/github/tasks/event_extractor.go +++ b/backend/plugins/github/tasks/event_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -54,21 +52,16 @@ type IssueEvent struct { func ExtractApiEvents(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[IssueEvent]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_EVENTS_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - body := &IssueEvent{} - err := errors.Convert(json.Unmarshal(row.Data, body)) - if err != nil { - return nil, err - } + Extract: func(body *IssueEvent, row *api.RawData) ([]any, errors.Error) { results := make([]interface{}, 0, 1) if body.GithubId == 0 || body.Actor == nil { return nil, nil @@ -80,22 +73,15 @@ func ExtractApiEvents(taskCtx plugin.SubTaskContext) errors.Error { Type: body.Event, GithubCreatedAt: body.GithubCreatedAt.ToTime(), } - if body.Actor != nil { githubIssueEvent.AuthorUsername = body.Actor.Login - githubAccount, err := convertAccount(body.Actor, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } results = append(results, githubAccount) } - - if err != nil { - return nil, err - } results = append(results, githubIssueEvent) - return results, nil }, }) diff --git a/backend/plugins/github/tasks/http_client.go b/backend/plugins/github/tasks/http_client.go new file mode 100644 index 00000000000..33ef6a3df6b --- /dev/null +++ b/backend/plugins/github/tasks/http_client.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/github/token" +) + +func CreateAuthenticatedHttpClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + baseClient *http.Client, +) (*http.Client, errors.Error) { + + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) + + if baseClient == nil { + baseClient = &http.Client{} + } + + // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. + var tp *token.TokenProvider + if connection.RefreshToken != "" { + tp = token.NewTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { + tp = token.NewAppInstallationTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } + + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + if tp != nil { + baseClient.Transport = token.NewRefreshRoundTripper(baseTransport, tp) + logger.Info( + "Installed token refresh round tripper for connection %d (authMethod=%s)", + connection.ID, + connection.AuthMethod, + ) + + } else if connection.Token != "" { + baseClient.Transport = token.NewStaticRoundTripper( + baseTransport, + connection.Token, + ) + logger.Info( + "Installed static token round tripper for connection %d", + connection.ID, + ) + } + + // Persist the freshly minted token so the DB has a correctly encrypted value. + // PrepareApiClient (called by NewApiClientFromConnection) mints the token + // in-memory but does not persist it; without this, the DB may contain a stale + // or corrupted token that breaks GET /connections. + if connection.AuthMethod == models.AppKey && connection.Token != "" { + if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { + logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) + } else { + logger.Info("Persisted initial token for connection %d", connection.ID) + } + } + + return baseClient, nil +} diff --git a/backend/plugins/github/tasks/issue_assignee_convertor.go b/backend/plugins/github/tasks/issue_assignee_convertor.go index 5f8c883b8f9..f632db8a6df 100644 --- a/backend/plugins/github/tasks/issue_assignee_convertor.go +++ b/backend/plugins/github/tasks/issue_assignee_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -53,31 +51,35 @@ func ConvertIssueAssignee(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubIssueAssignee{}), - dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - issueIdGen := didgen.NewDomainIdGenerator(&models.GithubIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubIssueAssignee]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_ISSUE_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_ISSUE_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubIssueAssignee{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubIssueAssignee := inputRow.(*models.GithubIssueAssignee) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubIssueAssignee{}), + dal.Where("_tool_github_issue_assignees.repo_id = ? and _tool_github_issue_assignees.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, + dal.Join(`left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_assignees.issue_id and _tool_github_issues.connection_id = _tool_github_issue_assignees.connection_id`), + ) + clauses = append(clauses, dal.Where("_tool_github_issues.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubIssueAssignee *models.GithubIssueAssignee) ([]interface{}, errors.Error) { issueAssignee := &ticket.IssueAssignee{ IssueId: issueIdGen.Generate(data.Options.ConnectionId, githubIssueAssignee.IssueId), AssigneeId: accountIdGen.Generate(data.Options.ConnectionId, githubIssueAssignee.AssigneeId), diff --git a/backend/plugins/github/tasks/issue_collector.go b/backend/plugins/github/tasks/issue_collector.go index 5558da69780..4b38d4be684 100644 --- a/backend/plugins/github/tasks/issue_collector.go +++ b/backend/plugins/github/tasks/issue_collector.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "net/url" + "time" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -78,7 +79,7 @@ func CollectApiIssues(taskCtx plugin.SubTaskContext) errors.Error { query := url.Values{} query.Set("state", "all") if apiCollector.GetSince() != nil { - query.Set("since", apiCollector.GetSince().String()) + query.Set("since", apiCollector.GetSince().UTC().Format(time.RFC3339)) } query.Set("direction", "asc") query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) diff --git a/backend/plugins/github/tasks/issue_comment_convertor.go b/backend/plugins/github/tasks/issue_comment_convertor.go index 07ddeec65c3..2510e32d2d6 100644 --- a/backend/plugins/github/tasks/issue_comment_convertor.go +++ b/backend/plugins/github/tasks/issue_comment_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -53,33 +51,34 @@ func ConvertIssueComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubIssueComment{}), - dal.Join("left join _tool_github_issues "+ - "on _tool_github_issues.github_id = _tool_github_issue_comments.issue_id"), - dal.Where("repo_id = ? and _tool_github_issues.connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - issueIdGen := didgen.NewDomainIdGenerator(&models.GithubIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubIssueComment{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubIssueComment]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_COMMENTS_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_COMMENTS_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubIssueComment := inputRow.(*models.GithubIssueComment) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubIssueComment{}), + dal.Join("left join _tool_github_issues " + + "on _tool_github_issues.github_id = _tool_github_issue_comments.issue_id"), + dal.Where("repo_id = ? and _tool_github_issues.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_issue_comments.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubIssueComment *models.GithubIssueComment) ([]interface{}, errors.Error) { domainIssueComment := &ticket.IssueComment{ DomainEntity: domainlayer.DomainEntity{ Id: issueIdGen.Generate(data.Options.ConnectionId, githubIssueComment.GithubId), diff --git a/backend/plugins/github/tasks/issue_convertor.go b/backend/plugins/github/tasks/issue_convertor.go index e50f6f22ac0..13e79529aec 100644 --- a/backend/plugins/github/tasks/issue_convertor.go +++ b/backend/plugins/github/tasks/issue_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "strconv" "strings" @@ -57,33 +56,33 @@ func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - issue := &models.GithubIssue{} - cursor, err := db.Cursor( - dal.From(issue), - dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - issueIdGen := didgen.NewDomainIdGenerator(&models.GithubIssue{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) boardIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubIssue]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_ISSUE_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_ISSUE_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubIssue{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - issue := inputRow.(*models.GithubIssue) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubIssue{}), + dal.Where("repo_id = ? and connection_id=?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(issue *models.GithubIssue) ([]interface{}, errors.Error) { domainIssue := &ticket.Issue{ DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(data.Options.ConnectionId, issue.GithubId)}, IssueKey: strconv.Itoa(issue.Number), diff --git a/backend/plugins/github/tasks/issue_extractor.go b/backend/plugins/github/tasks/issue_extractor.go index 101b1e5b9a3..43a276f3e58 100644 --- a/backend/plugins/github/tasks/issue_extractor.go +++ b/backend/plugins/github/tasks/issue_extractor.go @@ -22,6 +22,7 @@ import ( "regexp" "strings" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" @@ -84,44 +85,53 @@ type IssueRegexes struct { func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) + db := taskCtx.GetDal() config := data.Options.ScopeConfig issueRegexes, err := NewIssueRegexes(config) if err != nil { return nil } - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ + + extractor, extErr := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[IssuesResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_ISSUE_TABLE, + SubtaskConfig: map[string]string{ + "issueSeverity": config.IssueSeverity, + "issueComponent": config.IssueComponent, + "issuePriority": config.IssuePriority, + "issueTypeBug": config.IssueTypeBug, + "issueTypeRequirement": config.IssueTypeRequirement, + "issueTypeIncident": config.IssueTypeIncident, + }, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - body := &IssuesResponse{} - err := errors.Convert(json.Unmarshal(row.Data, body)) - if err != nil { - return nil, err + BeforeExtract: func(body *IssuesResponse, stateManager *api.SubtaskStateManager) errors.Error { + if stateManager.IsIncremental() { + if err := db.Delete(&models.GithubIssueLabel{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.GithubId), + ); err != nil { + return errors.Convert(err) + } + return errors.Convert(db.Delete(&models.GithubIssueAssignee{}, + dal.Where("connection_id = ? AND issue_id = ?", data.Options.ConnectionId, body.GithubId), + )) } - // need to extract 2 kinds of entities here + return nil + }, + Extract: func(body *IssuesResponse, row *api.RawData) ([]any, errors.Error) { if body.GithubId == 0 { return nil, nil } - //If this is a pr, ignore + // If this is a pr, ignore if body.PullRequest.Url != "" { return nil, nil } results := make([]interface{}, 0, 2) - githubIssue, err := convertGithubIssue(body, data.Options.ConnectionId, data.Options.GithubId) if err != nil { return nil, err @@ -163,8 +173,8 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error { return results, nil }, }) - if err != nil { - return err + if extErr != nil { + return extErr } return extractor.Execute() diff --git a/backend/plugins/github/tasks/issue_label_convertor.go b/backend/plugins/github/tasks/issue_label_convertor.go index bff0c0143db..6ef940b0601 100644 --- a/backend/plugins/github/tasks/issue_label_convertor.go +++ b/backend/plugins/github/tasks/issue_label_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -51,31 +49,33 @@ func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubIssueLabel{}), - dal.Join(`left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_labels.issue_id`), - dal.Where("_tool_github_issues.repo_id = ? and _tool_github_issues.connection_id = ?", repoId, data.Options.ConnectionId), - dal.Orderby("issue_id ASC"), - ) - if err != nil { - return err - } - defer cursor.Close() issueIdGen := didgen.NewDomainIdGenerator(&models.GithubIssue{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubIssueLabel]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_ISSUE_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_ISSUE_TABLE, }, - InputRowType: reflect.TypeOf(models.GithubIssueLabel{}), - Input: cursor, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - issueLabel := inputRow.(*models.GithubIssueLabel) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubIssueLabel{}), + dal.Join(`left join _tool_github_issues on _tool_github_issues.github_id = _tool_github_issue_labels.issue_id`), + dal.Where("_tool_github_issues.repo_id = ? and _tool_github_issues.connection_id = ?", repoId, data.Options.ConnectionId), + dal.Orderby("issue_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_issues.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(issueLabel *models.GithubIssueLabel) ([]interface{}, errors.Error) { domainIssueLabel := &ticket.IssueLabel{ IssueId: issueIdGen.Generate(data.Options.ConnectionId, issueLabel.IssueId), LabelName: issueLabel.LabelName, diff --git a/backend/plugins/github/tasks/milestone_extractor.go b/backend/plugins/github/tasks/milestone_extractor.go index 7828b2c26f9..4737bd47e82 100644 --- a/backend/plugins/github/tasks/milestone_extractor.go +++ b/backend/plugins/github/tasks/milestone_extractor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "encoding/json" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -81,24 +79,17 @@ type MilestonesResponse struct { func ExtractMilestones(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[MilestonesResponse]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_MILESTONE_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - response := &MilestonesResponse{} - err := errors.Convert(json.Unmarshal(row.Data, response)) - if err != nil { - return nil, err - } - results := make([]interface{}, 0, 1) - results = append(results, convertGithubMilestone(response, data.Options.ConnectionId, data.Options.GithubId)) - return results, nil + Extract: func(body *MilestonesResponse, row *api.RawData) ([]any, errors.Error) { + return []any{convertGithubMilestone(body, data.Options.ConnectionId, data.Options.GithubId)}, nil }, }) if err != nil { diff --git a/backend/plugins/github/tasks/pr_comment_convertor.go b/backend/plugins/github/tasks/pr_comment_convertor.go index 43e0e73ba2b..b02342e2b8f 100644 --- a/backend/plugins/github/tasks/pr_comment_convertor.go +++ b/backend/plugins/github/tasks/pr_comment_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -54,34 +52,36 @@ func ConvertPullRequestComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubPrComment{}), - dal.Join("left join _tool_github_pull_requests "+ - "on _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id"), - dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - prCommentIdGen := didgen.NewDomainIdGenerator(&models.GithubPrComment{}) prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) prReviewIdGen := didgen.NewDomainIdGenerator(&models.GithubPrReview{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPrComment{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPrComment]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_COMMENTS_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_COMMENTS_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubPullRequestComment := inputRow.(*models.GithubPrComment) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPrComment{}), + dal.Join("left join _tool_github_pull_requests " + + "on _tool_github_pull_requests.github_id = _tool_github_pull_request_comments.pull_request_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_request_comments.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPullRequestComment *models.GithubPrComment) ([]interface{}, errors.Error) { domainPrComment := &code.PullRequestComment{ DomainEntity: domainlayer.DomainEntity{ Id: prCommentIdGen.Generate(data.Options.ConnectionId, githubPullRequestComment.GithubId), diff --git a/backend/plugins/github/tasks/pr_commit_convertor.go b/backend/plugins/github/tasks/pr_commit_convertor.go index b9cbbbbf52b..d21c343dcad 100644 --- a/backend/plugins/github/tasks/pr_commit_convertor.go +++ b/backend/plugins/github/tasks/pr_commit_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -54,30 +52,31 @@ func ConvertPullRequestCommits(taskCtx plugin.SubTaskContext) (err errors.Error) pullIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) - cursor, err := db.Cursor( - dal.From(&models.GithubPrCommit{}), - dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_commits.pull_request_id`), - dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - dal.Orderby("pull_request_id ASC"), - ) - if err != nil { - return err - } - defer cursor.Close() - - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPrCommit{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPrCommit]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_COMMIT_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PR_COMMIT_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubPullRequestCommit := inputRow.(*models.GithubPrCommit) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPrCommit{}), + dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_commits.pull_request_id`), + dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + dal.Orderby("pull_request_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPullRequestCommit *models.GithubPrCommit) ([]interface{}, errors.Error) { domainPrCommit := &code.PullRequestCommit{ CommitSha: githubPullRequestCommit.CommitSha, PullRequestId: pullIdGen.Generate(data.Options.ConnectionId, githubPullRequestCommit.PullRequestId), diff --git a/backend/plugins/github/tasks/pr_commit_extractor.go b/backend/plugins/github/tasks/pr_commit_extractor.go index 569e39abac5..536af4d26e7 100644 --- a/backend/plugins/github/tasks/pr_commit_extractor.go +++ b/backend/plugins/github/tasks/pr_commit_extractor.go @@ -70,62 +70,44 @@ type PullRequestCommit struct { func ExtractApiPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ + extractor, err := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[PrCommitsResponse]{ + SubtaskCommonArgs: &helper.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_PR_COMMIT_TABLE, }, - Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - apiPullRequestCommit := &PrCommitsResponse{} + Extract: func(body *PrCommitsResponse, row *helper.RawData) ([]any, errors.Error) { if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") { return nil, nil } - err := errors.Convert(json.Unmarshal(row.Data, apiPullRequestCommit)) - if err != nil { - return nil, err - } pull := &SimplePr{} - err = errors.Convert(json.Unmarshal(row.Input, pull)) + err := errors.Convert(json.Unmarshal(row.Input, pull)) if err != nil { return nil, err } - // need to extract 2 kinds of entities here results := make([]interface{}, 0, 3) githubRepoCommit := &models.GithubRepoCommit{ ConnectionId: data.Options.ConnectionId, RepoId: repoId, - CommitSha: apiPullRequestCommit.Sha, + CommitSha: body.Sha, } results = append(results, githubRepoCommit) - - githubCommit, err := convertPullRequestCommit(apiPullRequestCommit, data.Options.ConnectionId) + githubCommit, err := convertPullRequestCommit(body, data.Options.ConnectionId) if err != nil { return nil, err } results = append(results, githubCommit) - githubPullRequestCommit := &models.GithubPrCommit{ ConnectionId: data.Options.ConnectionId, - CommitSha: apiPullRequestCommit.Sha, + CommitSha: body.Sha, PullRequestId: pull.GithubId, CommitAuthorName: githubCommit.AuthorName, CommitAuthorEmail: githubCommit.AuthorEmail, CommitAuthoredDate: githubCommit.AuthoredDate, } - if err != nil { - return nil, err - } results = append(results, githubPullRequestCommit) return results, nil }, diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 4a94d17dfe2..c4e27a6b9bb 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -53,32 +51,33 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubPullRequest{}), - dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) repoIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPullRequest{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPullRequest]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PULL_REQUEST_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PULL_REQUEST_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - pr := inputRow.(*models.GithubPullRequest) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPullRequest{}), + dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(pr *models.GithubPullRequest) ([]interface{}, errors.Error) { domainPr := &code.PullRequest{ DomainEntity: domainlayer.DomainEntity{ Id: prIdGen.Generate(data.Options.ConnectionId, pr.GithubId), @@ -88,7 +87,6 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { OriginalStatus: pr.State, Title: pr.Title, Url: pr.Url, - AuthorId: accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId), AuthorName: pr.AuthorName, Description: pr.Body, CreatedDate: pr.GithubCreatedAt, @@ -105,9 +103,17 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { Additions: pr.Additions, Deletions: pr.Deletions, MergedByName: pr.MergedByName, - MergedById: accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById), IsDraft: pr.IsDraft, } + // Generate account ids only for real users (#8886): a zero AuthorId (deleted + // user) or zero MergedById (unmerged PR) would otherwise produce an id like + // github:GithubAccount:1:0 that no accounts row can ever match. + if pr.AuthorId != 0 { + domainPr.AuthorId = accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId) + } + if pr.MergedById != 0 { + domainPr.MergedById = accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById) + } if pr.State == "open" || pr.State == "OPEN" { domainPr.Status = code.OPEN } else if pr.State == "MERGED" || (pr.State == "closed" && (pr.Merged || pr.MergedAt != nil)) { diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go index ea78583858b..9c3f1ca8fc9 100644 --- a/backend/plugins/github/tasks/pr_extractor.go +++ b/backend/plugins/github/tasks/pr_extractor.go @@ -21,6 +21,7 @@ import ( "encoding/json" "regexp" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" @@ -85,6 +86,7 @@ type GithubApiPullRequest struct { func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) + db := taskCtx.GetDal() config := data.Options.ScopeConfig var labelTypeRegex *regexp.Regexp var labelComponentRegex *regexp.Regexp @@ -104,45 +106,44 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { } } - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ + extractor, extErr := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[GithubApiPullRequest]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_PULL_REQUEST_TABLE, + SubtaskConfig: map[string]string{ + "prType": prType, + "prComponent": prComponent, + }, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - rawL := &GithubApiPullRequest{} - err := errors.Convert(json.Unmarshal(row.Data, rawL)) - if err != nil { - return nil, err + BeforeExtract: func(body *GithubApiPullRequest, stateManager *api.SubtaskStateManager) errors.Error { + if stateManager.IsIncremental() { + return errors.Convert(db.Delete( + &models.GithubPrLabel{}, + dal.Where("connection_id = ? AND pull_id = ?", data.Options.ConnectionId, body.GithubId), + )) } - // need to extract 2 kinds of entities here + return nil + }, + Extract: func(body *GithubApiPullRequest, row *api.RawData) ([]any, errors.Error) { results := make([]interface{}, 0, 1) - if rawL.GithubId == 0 { + if body.GithubId == 0 { return nil, nil } // Filter bot PRs by username - if rawL.User != nil && shouldSkipByUsername(rawL.User.Login) { - taskCtx.GetLogger().Debug("Skipping PR #%d from bot user: %s", rawL.Number, rawL.User.Login) + if body.User != nil && shouldSkipByUsername(body.User.Login) { + taskCtx.GetLogger().Debug("Skipping PR #%d from bot user: %s", body.Number, body.User.Login) return nil, nil } - //If this is a pr, ignore - githubPr, err := convertGithubPullRequest(rawL, data.Options.ConnectionId, data.Options.GithubId) + githubPr, err := convertGithubPullRequest(body, data.Options.ConnectionId, data.Options.GithubId) if err != nil { return nil, err } - if rawL.User != nil { - githubUser, err := convertAccount(rawL.User, data.Options.GithubId, data.Options.ConnectionId) + if body.User != nil { + githubUser, err := convertAccount(body.User, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } @@ -150,33 +151,41 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { githubPr.AuthorName = githubUser.Login githubPr.AuthorId = githubUser.AccountId } - for _, label := range rawL.Labels { + // Emit a repo_account for the merged-by user too, so pull_requests.merged_by_id + // resolves to a domain account instead of an orphan FK (ConvertAccounts sources + // every referenced user from _tool_github_repo_accounts). + if body.MergedBy != nil { + mergedByUser, err := convertAccount(body.MergedBy, data.Options.GithubId, data.Options.ConnectionId) + if err != nil { + return nil, err + } + results = append(results, mergedByUser) + } + for _, label := range body.Labels { results = append(results, &models.GithubPrLabel{ ConnectionId: data.Options.ConnectionId, PullId: githubPr.GithubId, LabelName: label.Name, }) - // if pr.Type has not been set and prType is set in .env, process the below if labelTypeRegex != nil && labelTypeRegex.MatchString(label.Name) { githubPr.Type = label.Name } - // if pr.Component has not been set and prComponent is set in .env, process if labelComponentRegex != nil && labelComponentRegex.MatchString(label.Name) { githubPr.Component = label.Name } } results = append(results, githubPr) - return results, nil }, }) - if err != nil { - return errors.Default.Wrap(err, "error initializing Github PR extractor") + if extErr != nil { + return errors.Default.Wrap(extErr, "error initializing Github PR extractor") } return extractor.Execute() } + func convertGithubPullRequest(pull *GithubApiPullRequest, connId uint64, repoId int) (*models.GithubPullRequest, errors.Error) { githubPull := &models.GithubPullRequest{ ConnectionId: connId, diff --git a/backend/plugins/github/tasks/pr_issue_convertor.go b/backend/plugins/github/tasks/pr_issue_convertor.go index 979a8b128f3..0d724224f77 100644 --- a/backend/plugins/github/tasks/pr_issue_convertor.go +++ b/backend/plugins/github/tasks/pr_issue_convertor.go @@ -19,7 +19,6 @@ package tasks import ( "github.com/spf13/cast" - "reflect" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -52,32 +51,34 @@ func ConvertPullRequestIssues(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubPrIssue{}), - dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_issues.pull_request_id`), - dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - dal.Orderby("pull_request_id ASC"), - ) - if err != nil { - return err - } - defer cursor.Close() prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) issueIdGen := didgen.NewDomainIdGenerator(&models.GithubIssue{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPrIssue{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPrIssue]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PULL_REQUEST_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PULL_REQUEST_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubPrIssue := inputRow.(*models.GithubPrIssue) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPrIssue{}), + dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_issues.pull_request_id`), + dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + dal.Orderby("pull_request_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPrIssue *models.GithubPrIssue) ([]interface{}, errors.Error) { pullRequestIssue := &crossdomain.PullRequestIssue{ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPrIssue.PullRequestId), IssueId: issueIdGen.Generate(data.Options.ConnectionId, githubPrIssue.IssueId), diff --git a/backend/plugins/github/tasks/pr_issue_enricher.go b/backend/plugins/github/tasks/pr_issue_enricher.go index bf4d2837d17..cf8228b79a3 100644 --- a/backend/plugins/github/tasks/pr_issue_enricher.go +++ b/backend/plugins/github/tasks/pr_issue_enricher.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "reflect" "regexp" "strconv" "strings" @@ -62,27 +61,42 @@ func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext) (err errors.Error) { } } charPattern := regexp.MustCompile(`[\/a-zA-Z\s,]+`) - cursor, err := db.Cursor(dal.From(&models.GithubPullRequest{}), - dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId)) - if err != nil { - return err - } - defer cursor.Close() - // iterate all rows - - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPullRequest{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPullRequest]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PULL_REQUEST_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PULL_REQUEST_TABLE, + SubtaskConfig: map[string]string{ + "prBodyClosePattern": prBodyClosePattern, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPullRequest{}), + dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + BeforeConvert: func(githubPullRequest *models.GithubPullRequest, stateManager *api.SubtaskStateManager) errors.Error { + if !stateManager.IsIncremental() { + return nil + } + return db.Delete( + &models.GithubPrIssue{}, + dal.Where("connection_id = ? AND pull_request_id = ?", data.Options.ConnectionId, githubPullRequest.GithubId), + ) }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubPullRequst := inputRow.(*models.GithubPullRequest) + Convert: func(githubPullRequst *models.GithubPullRequest) ([]interface{}, errors.Error) { results := make([]interface{}, 0, 1) //find the matched string in body diff --git a/backend/plugins/github/tasks/pr_label_convertor.go b/backend/plugins/github/tasks/pr_label_convertor.go index 3351218a71f..432a7308f27 100644 --- a/backend/plugins/github/tasks/pr_label_convertor.go +++ b/backend/plugins/github/tasks/pr_label_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -51,31 +49,33 @@ func ConvertPullRequestLabels(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubPrLabel{}), - dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_labels.pull_id`), - dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - dal.Orderby("pull_id ASC"), - ) - if err != nil { - return err - } - defer cursor.Close() prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPrLabel{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPrLabel]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PULL_REQUEST_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PULL_REQUEST_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - prLabel := inputRow.(*models.GithubPrLabel) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPrLabel{}), + dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_labels.pull_id`), + dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + dal.Orderby("pull_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(prLabel *models.GithubPrLabel) ([]interface{}, errors.Error) { domainPrLabel := &code.PullRequestLabel{ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, prLabel.PullId), LabelName: prLabel.LabelName, diff --git a/backend/plugins/github/tasks/pr_review_comment_collector.go b/backend/plugins/github/tasks/pr_review_comment_collector.go index d6dfb296f42..ac29938cf1e 100644 --- a/backend/plugins/github/tasks/pr_review_comment_collector.go +++ b/backend/plugins/github/tasks/pr_review_comment_collector.go @@ -22,6 +22,7 @@ import ( "fmt" "net/http" "net/url" + "time" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -75,7 +76,7 @@ func CollectPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error { Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { query := url.Values{} if apiCollector.GetSince() != nil { - query.Set("since", apiCollector.GetSince().String()) + query.Set("since", apiCollector.GetSince().UTC().Format(time.RFC3339)) } query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("direction", "asc") diff --git a/backend/plugins/github/tasks/pr_review_comment_extractor.go b/backend/plugins/github/tasks/pr_review_comment_extractor.go index ed8aa182584..cc39806cdba 100644 --- a/backend/plugins/github/tasks/pr_review_comment_extractor.go +++ b/backend/plugins/github/tasks/pr_review_comment_extractor.go @@ -48,86 +48,75 @@ var ExtractApiPrReviewCommentsMeta = plugin.SubTaskMeta{ models.GithubPrCommit{}.TableName()}, } +type GithubPrReviewCommentBody struct { + GithubId int `json:"id"` + Body json.RawMessage + User *GithubAccountResponse + PrUrl string `json:"pull_request_url"` + GithubCreatedAt common.Iso8601Time `json:"created_at"` + GithubUpdatedAt common.Iso8601Time `json:"updated_at"` + CommitId string `json:"commit_id"` + PrReviewId int `json:"pull_request_review_id"` +} + func ExtractApiPrReviewComments(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) db := taskCtx.GetDal() + prUrlPattern := fmt.Sprintf(`https\:\/\/api\.github\.com\/repos\/%s\/pulls\/(\d+)`, data.Options.Name) + prUrlRegex, err := regexp.Compile(prUrlPattern) + if err != nil { + return errors.Default.Wrap(err, "regexp Compile prUrlPattern failed") + } - extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ - RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ - Ctx: taskCtx, + extractor, extErr := helper.NewStatefulApiExtractor(&helper.StatefulApiExtractorArgs[GithubPrReviewCommentBody]{ + SubtaskCommonArgs: &helper.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, Table: RAW_PR_REVIEW_COMMENTS_TABLE, }, - Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { - var prReviewComment struct { - GithubId int `json:"id"` - Body json.RawMessage - User *GithubAccountResponse - PrUrl string `json:"pull_request_url"` - GithubCreatedAt common.Iso8601Time `json:"created_at"` - GithubUpdatedAt common.Iso8601Time `json:"updated_at"` - CommitId string `json:"commit_id"` - PrReviewId int `json:"pull_request_review_id"` - } - err := errors.Convert(json.Unmarshal(row.Data, &prReviewComment)) - if err != nil { - return nil, err - } - + Extract: func(body *GithubPrReviewCommentBody, row *helper.RawData) ([]any, errors.Error) { results := make([]interface{}, 0, 1) githubPrComment := &models.GithubPrComment{ ConnectionId: data.Options.ConnectionId, - GithubId: prReviewComment.GithubId, - Body: string(prReviewComment.Body), - CommitSha: prReviewComment.CommitId, - ReviewId: prReviewComment.PrReviewId, - GithubCreatedAt: prReviewComment.GithubCreatedAt.ToTime(), - GithubUpdatedAt: prReviewComment.GithubUpdatedAt.ToTime(), + GithubId: body.GithubId, + Body: string(body.Body), + CommitSha: body.CommitId, + ReviewId: body.PrReviewId, + GithubCreatedAt: body.GithubCreatedAt.ToTime(), + GithubUpdatedAt: body.GithubUpdatedAt.ToTime(), Type: "DIFF", } - - prUrlRegex, err := errors.Convert01(regexp.Compile(prUrlPattern)) - if err != nil { - return nil, errors.Default.Wrap(err, "regexp Compile prUrlPattern failed") - } - prId, err := enrichGithubPrComment(data, db, prUrlRegex, prReviewComment.PrUrl) + prId, err := enrichGithubPrComment(data, db, prUrlRegex, body.PrUrl) if err != nil { return nil, errors.Default.Wrap(err, "parse prId failed") } if prId != 0 { githubPrComment.PullRequestId = prId } - - if prReviewComment.User != nil { - // Filter bot comments by username - if shouldSkipByUsername(prReviewComment.User.Login) { - taskCtx.GetLogger().Debug("Skipping PR review comment #%d from bot user: %s", prReviewComment.GithubId, prReviewComment.User.Login) + if body.User != nil { + if shouldSkipByUsername(body.User.Login) { + taskCtx.GetLogger().Debug("Skipping PR review comment #%d from bot user: %s", body.GithubId, body.User.Login) return nil, nil } - - githubPrComment.AuthorUserId = prReviewComment.User.Id - githubPrComment.AuthorUsername = prReviewComment.User.Login - - githubAccount, err := convertAccount(prReviewComment.User, data.Options.GithubId, data.Options.ConnectionId) + githubPrComment.AuthorUserId = body.User.Id + githubPrComment.AuthorUsername = body.User.Login + githubAccount, err := convertAccount(body.User, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } results = append(results, githubAccount) } - results = append(results, githubPrComment) return results, nil }, }) - - if err != nil { - return err + if extErr != nil { + return extErr } - return extractor.Execute() } diff --git a/backend/plugins/github/tasks/pr_review_convertor.go b/backend/plugins/github/tasks/pr_review_convertor.go index 4ff2f353126..f266b130a37 100644 --- a/backend/plugins/github/tasks/pr_review_convertor.go +++ b/backend/plugins/github/tasks/pr_review_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -53,34 +51,35 @@ func ConvertPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubPrReview{}), - dal.Join("left join _tool_github_pull_requests "+ - "on _tool_github_pull_requests.github_id = _tool_github_pull_request_reviews.pull_request_id"), - dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - prReviewUIdGen := didgen.NewDomainIdGenerator(&models.GithubPrReview{}) prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubPrReview{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubPrReview]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_REVIEW_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PR_REVIEW_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubPullRequestReview := inputRow.(*models.GithubPrReview) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubPrReview{}), + dal.Join("left join _tool_github_pull_requests " + + "on _tool_github_pull_requests.github_id = _tool_github_pull_request_reviews.pull_request_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPullRequestReview *models.GithubPrReview) ([]interface{}, errors.Error) { domainPrReview := &code.PullRequestComment{ DomainEntity: domainlayer.DomainEntity{ Id: prReviewUIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.GithubId), diff --git a/backend/plugins/github/tasks/pr_review_extractor.go b/backend/plugins/github/tasks/pr_review_extractor.go index 2af41b60a7e..ff8015411ee 100644 --- a/backend/plugins/github/tasks/pr_review_extractor.go +++ b/backend/plugins/github/tasks/pr_review_extractor.go @@ -56,79 +56,59 @@ type PullRequestReview struct { func ExtractApiPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) - extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ + extractor, err := api.NewStatefulApiExtractor(&api.StatefulApiExtractorArgs[PullRequestReview]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - /* - Table store raw data - */ Table: RAW_PR_REVIEW_TABLE, }, - Extract: func(row *api.RawData) ([]interface{}, errors.Error) { - apiPullRequestReview := &PullRequestReview{} + Extract: func(body *PullRequestReview, row *api.RawData) ([]any, errors.Error) { if strings.HasPrefix(string(row.Data), "{\"message\": \"Not Found\"") { return nil, nil } - err := errors.Convert(json.Unmarshal(row.Data, apiPullRequestReview)) - if err != nil { - return nil, err - } - if apiPullRequestReview.State == "PENDING" || apiPullRequestReview.User == nil { + if body.State == "PENDING" || body.User == nil { return nil, nil } // Filter bot reviews by username - if shouldSkipByUsername(apiPullRequestReview.User.Login) { - taskCtx.GetLogger().Debug("Skipping review #%d from bot user: %s", apiPullRequestReview.GithubId, apiPullRequestReview.User.Login) + if shouldSkipByUsername(body.User.Login) { + taskCtx.GetLogger().Debug("Skipping review #%d from bot user: %s", body.GithubId, body.User.Login) return nil, nil } pull := &SimplePr{} - err = errors.Convert(json.Unmarshal(row.Input, pull)) + err := errors.Convert(json.Unmarshal(row.Input, pull)) if err != nil { return nil, err } - // need to extract 2 kinds of entities here results := make([]interface{}, 0, 1) - githubReviewer := &models.GithubReviewer{ ConnectionId: data.Options.ConnectionId, PullRequestId: pull.GithubId, } - githubPrReview := &models.GithubPrReview{ ConnectionId: data.Options.ConnectionId, - GithubId: apiPullRequestReview.GithubId, - Body: apiPullRequestReview.Body, - State: apiPullRequestReview.State, - CommitSha: apiPullRequestReview.CommitId, - GithubSubmitAt: apiPullRequestReview.SubmittedAt.ToNullableTime(), + GithubId: body.GithubId, + Body: body.Body, + State: body.State, + CommitSha: body.CommitId, + GithubSubmitAt: body.SubmittedAt.ToNullableTime(), PullRequestId: pull.GithubId, } - - if apiPullRequestReview.User != nil { - githubReviewer.ReviewerId = apiPullRequestReview.User.Id - githubReviewer.Username = apiPullRequestReview.User.Login - - githubPrReview.AuthorUserId = apiPullRequestReview.User.Id - githubPrReview.AuthorUsername = apiPullRequestReview.User.Login - - githubUser, err := convertAccount(apiPullRequestReview.User, data.Options.GithubId, data.Options.ConnectionId) + if body.User != nil { + githubReviewer.ReviewerId = body.User.Id + githubReviewer.Username = body.User.Login + githubPrReview.AuthorUserId = body.User.Id + githubPrReview.AuthorUsername = body.User.Login + githubUser, err := convertAccount(body.User, data.Options.GithubId, data.Options.ConnectionId) if err != nil { return nil, err } results = append(results, githubUser) } - results = append(results, githubReviewer) results = append(results, githubPrReview) - return results, nil }, }) diff --git a/backend/plugins/github/tasks/release_convertor.go b/backend/plugins/github/tasks/release_convertor.go index be49141eb5d..a5c2f941d9f 100644 --- a/backend/plugins/github/tasks/release_convertor.go +++ b/backend/plugins/github/tasks/release_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -50,28 +48,38 @@ var ConvertReleasesMeta = plugin.SubTaskMeta{ func ConvertRelease(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() - rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_RELEASE_TABLE) - cursor, err := db.Cursor( - dal.From(&models.GithubRelease{}), - dal.Where( - "published_at IS NOT NULL AND connection_id = ? AND github_id = ?", - data.Options.ConnectionId, data.Options.GithubId, - ), - ) - if err != nil { - return err + data := taskCtx.GetData().(*GithubTaskData) + params := GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, } - defer cursor.Close() releaseIdGen := didgen.NewDomainIdGenerator(&models.GithubRelease{}) releaseScopeIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubRelease{}), - Input: cursor, - RawDataSubTaskArgs: *rawDataSubTaskArgs, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubRelease := inputRow.(*models.GithubRelease) + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubRelease]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_RELEASE_TABLE, + Params: params, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubRelease{}), + dal.Where( + "published_at IS NOT NULL AND connection_id = ? AND github_id = ?", + data.Options.ConnectionId, data.Options.GithubId, + ), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubRelease *models.GithubRelease) ([]interface{}, errors.Error) { release := &devops.CicdRelease{ DomainEntity: domainlayer.DomainEntity{ Id: releaseIdGen.Generate(githubRelease.ConnectionId, githubRelease.Id), @@ -87,18 +95,12 @@ func ConvertRelease(taskCtx plugin.SubTaskContext) errors.Error { IsPrerelease: githubRelease.IsPrerelease, TagName: githubRelease.TagName, CommitSha: githubRelease.CommitSha, - - AuthorID: githubRelease.AuthorID, - - RepoId: releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId), + AuthorID: githubRelease.AuthorID, + RepoId: releaseScopeIdGen.Generate(githubRelease.ConnectionId, githubRelease.GithubId), } - - return []interface{}{ - release, - }, nil + return []interface{}{release}, nil }, }) - if err != nil { return err } diff --git a/backend/plugins/github/tasks/repo_convertor.go b/backend/plugins/github/tasks/repo_convertor.go index b9c711ad9f8..c6bff20b320 100644 --- a/backend/plugins/github/tasks/repo_convertor.go +++ b/backend/plugins/github/tasks/repo_convertor.go @@ -19,7 +19,6 @@ package tasks import ( "fmt" - "reflect" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -35,10 +34,6 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) -func init() { - RegisterSubtaskMeta(&ConvertRepoMeta) -} - type GithubApiRepo struct { Name string `json:"name"` FullName string `json:"full_name"` @@ -53,6 +48,10 @@ type GithubApiRepo struct { CloneUrl string `json:"clone_url"` } +func init() { + RegisterSubtaskMeta(&ConvertRepoMeta) +} + var ConvertRepoMeta = plugin.SubTaskMeta{ Name: "Convert Repos", EntryPoint: ConvertRepo, @@ -80,30 +79,31 @@ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubRepo{}), - dal.Where("github_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - repoIdGen := didgen.NewDomainIdGenerator(&models.GithubRepo{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubRepo{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubRepo]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: models.GithubRepo{}.TableName(), Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: models.GithubRepo{}.TableName(), }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - repository := inputRow.(*models.GithubRepo) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubRepo{}), + dal.Where("github_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(repository *models.GithubRepo) ([]interface{}, errors.Error) { domainRepository := &code.Repo{ DomainEntity: domainlayer.DomainEntity{ Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), @@ -125,12 +125,10 @@ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { Description: repository.Description, CreatedDate: repository.CreatedDate, } - domainBoardRepo := &crossdomain.BoardRepo{ BoardId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), RepoId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), } - domainCicdScope := &devops.CicdScope{ DomainEntity: domainlayer.DomainEntity{ Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), @@ -141,7 +139,6 @@ func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { CreatedDate: repository.CreatedDate, UpdatedDate: repository.UpdatedDate, } - return []interface{}{ domainRepository, domainBoard, diff --git a/backend/plugins/github/tasks/review_convertor.go b/backend/plugins/github/tasks/review_convertor.go index d4d10104256..18a0de211e0 100644 --- a/backend/plugins/github/tasks/review_convertor.go +++ b/backend/plugins/github/tasks/review_convertor.go @@ -18,8 +18,6 @@ limitations under the License. package tasks import ( - "reflect" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/code" @@ -52,33 +50,34 @@ func ConvertReviews(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*GithubTaskData) repoId := data.Options.GithubId - cursor, err := db.Cursor( - dal.From(&models.GithubReviewer{}), - dal.Join("left join _tool_github_pull_requests "+ - "on _tool_github_pull_requests.github_id = _tool_github_reviewers.pull_request_id"), - dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), - ) - if err != nil { - return err - } - defer cursor.Close() - prIdGen := didgen.NewDomainIdGenerator(&models.GithubPullRequest{}) accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewDataConverter(api.DataConverterArgs{ - InputRowType: reflect.TypeOf(models.GithubReviewer{}), - Input: cursor, - RawDataSubTaskArgs: api.RawDataSubTaskArgs{ - Ctx: taskCtx, + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubReviewer]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_REVIEW_TABLE, Params: GithubApiParams{ ConnectionId: data.Options.ConnectionId, Name: data.Options.Name, }, - Table: RAW_PR_REVIEW_TABLE, }, - Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { - githubReview := inputRow.(*models.GithubReviewer) + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&models.GithubReviewer{}), + dal.Join("left join _tool_github_pull_requests " + + "on _tool_github_pull_requests.github_id = _tool_github_reviewers.pull_request_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubReview *models.GithubReviewer) ([]interface{}, errors.Error) { domainReview := &code.PullRequestReviewer{ PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubReview.PullRequestId), ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, githubReview.ReviewerId), diff --git a/backend/plugins/github/token/app_installation_refresh.go b/backend/plugins/github/token/app_installation_refresh.go new file mode 100644 index 00000000000..2f4cf04b898 --- /dev/null +++ b/backend/plugins/github/token/app_installation_refresh.go @@ -0,0 +1,152 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +// tokenPrefix returns the first n characters of a token for safe logging. +func tokenPrefix(token string, n int) string { + if len(token) <= n { + return token + } + return token[:n] + "..." +} + +func refreshGitHubAppInstallationToken(tp *TokenProvider) errors.Error { + if tp == nil || tp.conn == nil { + return errors.Default.New("missing github connection for app token refresh") + } + if tp.conn.AuthMethod != models.AppKey || tp.conn.InstallationID == 0 { + return errors.Default.New("invalid github app connection for token refresh") + } + if tp.conn.Endpoint == "" { + return errors.Default.New("missing github endpoint for token refresh") + } + + oldToken := tp.conn.Token + if tp.logger != nil { + expiresStr := "unknown" + if tp.conn.TokenExpiresAt != nil { + expiresStr = tp.conn.TokenExpiresAt.Format(time.RFC3339) + } + tp.logger.Info( + "Refreshing GitHub App installation token for connection %d (installation %d), old token=%s, expires_at=%s", + tp.conn.ID, tp.conn.InstallationID, + tokenPrefix(oldToken, 8), + expiresStr, + ) + } + + // Use baseTransport (the unwrapped transport) to avoid deadlock. + // The httpClient's transport may be the RefreshRoundTripper which would + // re-enter GetToken() and deadlock on the mutex. + apiClient := newRefreshApiClientWithTransport(tp.conn.Endpoint, tp.baseTransport) + installationToken, err := tp.conn.GithubAppKey.GetInstallationAccessToken(apiClient) + if err != nil { + if tp.logger != nil { + tp.logger.Error(err, "Failed to refresh GitHub App installation token for connection %d", tp.conn.ID) + } + return err + } + + var expiresAt *time.Time + if !installationToken.ExpiresAt.IsZero() { + expiresAt = &installationToken.ExpiresAt + } + tp.conn.UpdateToken(installationToken.Token, "", expiresAt, nil) + + if tp.logger != nil { + tp.logger.Info( + "Successfully refreshed GitHub App installation token for connection %d, new token=%s, new expires_at=%s", + tp.conn.ID, + tokenPrefix(installationToken.Token, 8), + installationToken.ExpiresAt.Format(time.RFC3339), + ) + } + + persistAppToken(tp.dal, tp.conn, tp.encryptionSecret, tp.logger) + return nil +} + +func persistAppToken(d dal.Dal, conn *models.GithubConnection, encryptionSecret string, logger log.Logger) { + if d == nil || conn == nil { + return + } + if err := PersistEncryptedTokenColumns(d, conn, encryptionSecret, logger, false); err != nil { + if logger != nil { + logger.Warn(err, "Failed to persist refreshed app installation token for connection %d", conn.ID) + } + } else if logger != nil { + logger.Info("Persisted refreshed app installation token for connection %d", conn.ID) + } +} + +// PersistEncryptedTokenColumns manually encrypts token fields and writes them +// to the DB using UpdateColumns (map-based), which only touches the specified +// columns. This avoids two problems: +// - dal.Update (GORM Save) writes ALL fields, including refresh_token_expires_at +// which may have Go zero time that MySQL rejects as '0000-00-00'. +// - dal.UpdateColumns with plaintext bypasses the GORM encdec serializer, +// writing unencrypted tokens that corrupt subsequent reads. +// +// IMPORTANT: We pass the table name string (not the conn struct) to UpdateColumns +// so that GORM uses Table() instead of Model(). When Model(conn) is used, GORM +// processes the encdec serializer on the struct's Token field during statement +// preparation, which overwrites conn.Token in memory with the encrypted ciphertext. +// This corrupts the in-memory token causing immediate 401s on the next API call. +// +// If includeRefreshToken is true, refresh_token and refresh_token_expires_at +// are also written (used by the OAuth refresh path where these values are valid). +func PersistEncryptedTokenColumns(d dal.Dal, conn *models.GithubConnection, encryptionSecret string, logger log.Logger, includeRefreshToken bool) errors.Error { + encToken, err := plugin.Encrypt(encryptionSecret, conn.Token) + if err != nil { + return errors.Default.Wrap(err, "failed to encrypt token for persistence") + } + + sets := []dal.DalSet{ + {ColumnName: "token", Value: encToken}, + {ColumnName: "token_expires_at", Value: conn.TokenExpiresAt}, + } + + if includeRefreshToken { + encRefreshToken, err := plugin.Encrypt(encryptionSecret, conn.RefreshToken) + if err != nil { + return errors.Default.Wrap(err, "failed to encrypt refresh_token for persistence") + } + sets = append(sets, + dal.DalSet{ColumnName: "refresh_token", Value: encRefreshToken}, + dal.DalSet{ColumnName: "refresh_token_expires_at", Value: conn.RefreshTokenExpiresAt}, + ) + } + + // Use the table name string instead of the conn struct to prevent GORM from + // running the encdec serializer on conn.Token during Model() processing. + return d.UpdateColumns( + conn.TableName(), + sets, + dal.Where("id = ?", conn.ID), + ) +} diff --git a/backend/plugins/github/token/cmd/test_refresh/main.go b/backend/plugins/github/token/cmd/test_refresh/main.go new file mode 100644 index 00000000000..be1854dcc29 --- /dev/null +++ b/backend/plugins/github/token/cmd/test_refresh/main.go @@ -0,0 +1,240 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This script lists GitHub App installations and tests the token refresh flow. +// +// Usage: +// +// GITHUB_APP_ID=123456 GITHUB_APP_PEM="$(cat private-key.pem)" go run ./plugins/github/token/cmd/test_refresh/ +// +// Or if the key is in a file: +// +// GITHUB_APP_ID=123456 GITHUB_APP_PEM_FILE=/path/to/private-key.pem go run ./plugins/github/token/cmd/test_refresh/ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +type installation struct { + ID int `json:"id"` + Account struct { + Login string `json:"login"` + } `json:"account"` + AppID int `json:"app_id"` +} + +type installationToken struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` +} + +func main() { + appID := os.Getenv("GITHUB_APP_ID") + if appID == "" { + fatal("GITHUB_APP_ID env var is required") + } + + pemData := os.Getenv("GITHUB_APP_PEM") + if pemData == "" { + pemFile := os.Getenv("GITHUB_APP_PEM_FILE") + if pemFile == "" { + fatal("Set GITHUB_APP_PEM (contents) or GITHUB_APP_PEM_FILE (path)") + } + data, err := os.ReadFile(pemFile) + if err != nil { + fatal("failed to read PEM file: %v", err) + } + pemData = string(data) + } + + // Step 1: Create a JWT signed with the app's private key + fmt.Println("=== Step 1: Creating JWT from App ID and private key ===") + jwtToken, err := createJWT(appID, pemData) + if err != nil { + fatal("failed to create JWT: %v", err) + } + fmt.Printf("JWT created (first 20 chars): %s...\n\n", jwtToken[:20]) + + // Step 2: List installations + fmt.Println("=== Step 2: Listing installations for this app ===") + installations, err := listInstallations(jwtToken) + if err != nil { + fatal("failed to list installations: %v", err) + } + if len(installations) == 0 { + fatal("no installations found for this app") + } + for _, inst := range installations { + fmt.Printf(" Installation ID: %d Account: %s\n", inst.ID, inst.Account.Login) + } + fmt.Println() + + // Step 3: Get an installation token for the first installation + inst := installations[0] + fmt.Printf("=== Step 3: Minting installation token for %s (ID: %d) ===\n", inst.Account.Login, inst.ID) + token1, err := getInstallationToken(jwtToken, inst.ID) + if err != nil { + fatal("failed to get installation token: %v", err) + } + fmt.Printf("Token 1: %s... Expires: %s\n\n", token1.Token[:10], token1.ExpiresAt.Format(time.RFC3339)) + + // Step 4: Make an API call with the token to verify it works + fmt.Println("=== Step 4: Verifying token works (GET /installation/repositories) ===") + err = verifyToken(token1.Token) + if err != nil { + fatal("token verification failed: %v", err) + } + fmt.Printf("Token is valid and working.\n\n") + + // Step 5: Simulate the refresh flow — mint a second token (as our refreshFn would) + fmt.Println("=== Step 5: Simulating token refresh (minting a second token) ===") + jwtToken2, err := createJWT(appID, pemData) + if err != nil { + fatal("failed to create second JWT: %v", err) + } + token2, err := getInstallationToken(jwtToken2, inst.ID) + if err != nil { + fatal("failed to get second installation token: %v", err) + } + fmt.Printf("Token 2: %s... Expires: %s\n", token2.Token[:10], token2.ExpiresAt.Format(time.RFC3339)) + + if token1.Token == token2.Token { + fmt.Println("Note: Both tokens are identical (GitHub may cache short-lived tokens)") + } else { + fmt.Println("Tokens are different — refresh produced a new token.") + } + fmt.Println() + + // Step 6: Verify the new token works + fmt.Println("=== Step 6: Verifying refreshed token works ===") + err = verifyToken(token2.Token) + if err != nil { + fatal("refreshed token verification failed: %v", err) + } + fmt.Println("Refreshed token is valid and working.") + + fmt.Println("\n=== All steps passed. The token refresh flow works correctly. ===") + fmt.Printf("\nFor reference, your Installation ID is: %d\n", inst.ID) +} + +func createJWT(appID, pemData string) (string, error) { + privateKey, err := jwt.ParseRSAPrivateKeyFromPEM([]byte(pemData)) + if err != nil { + return "", fmt.Errorf("invalid PEM key: %w", err) + } + + now := time.Now().Unix() + token := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iat": now, + "exp": now + (10 * 60), // 10 minutes + "iss": appID, + }) + + signed, err := token.SignedString(privateKey) + if err != nil { + return "", fmt.Errorf("failed to sign JWT: %w", err) + } + return signed, nil +} + +func listInstallations(jwtToken string) ([]installation, error) { + req, err := http.NewRequest("GET", "https://api.github.com/app/installations", nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+jwtToken) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + var result []installation + if err := json.Unmarshal(body, &result); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + return result, nil +} + +func getInstallationToken(jwtToken string, installationID int) (*installationToken, error) { + url := fmt.Sprintf("https://api.github.com/app/installations/%d/access_tokens", installationID) + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+jwtToken) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusCreated { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + + var token installationToken + if err := json.Unmarshal(body, &token); err != nil { + return nil, fmt.Errorf("failed to parse token response: %w", err) + } + return &token, nil +} + +func verifyToken(token string) error { + req, err := http.NewRequest("GET", "https://api.github.com/installation/repositories?per_page=1", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(body)) + } + fmt.Printf(" HTTP 200 OK (X-RateLimit-Remaining: %s)\n", resp.Header.Get("X-RateLimit-Remaining")) + return nil +} + +func fatal(format string, args ...interface{}) { + fmt.Fprintf(os.Stderr, "FATAL: "+format+"\n", args...) + os.Exit(1) +} diff --git a/backend/plugins/github/token/refresh_api_client.go b/backend/plugins/github/token/refresh_api_client.go new file mode 100644 index 00000000000..1db6f691712 --- /dev/null +++ b/backend/plugins/github/token/refresh_api_client.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type refreshApiClient struct { + endpoint string + client *http.Client + timeout time.Duration +} + +// newRefreshApiClientWithTransport creates a refreshApiClient using a specific +// http.RoundTripper instead of an existing *http.Client. This is critical for +// avoiding deadlocks: the caller's *http.Client may have a RefreshRoundTripper +// transport that re-enters GetToken() and deadlocks on the mutex. +func newRefreshApiClientWithTransport(endpoint string, transport http.RoundTripper) plugin.ApiClient { + if transport == nil { + transport = http.DefaultTransport + } + return &refreshApiClient{ + endpoint: endpoint, + client: &http.Client{Transport: transport}, + timeout: 10 * time.Second, + } +} + +func (c *refreshApiClient) SetData(name string, data interface{}) {} + +func (c *refreshApiClient) GetData(name string) interface{} { return nil } + +func (c *refreshApiClient) SetHeaders(headers map[string]string) {} + +func (c *refreshApiClient) SetBeforeFunction(callback plugin.ApiClientBeforeRequest) {} + +func (c *refreshApiClient) GetBeforeFunction() plugin.ApiClientBeforeRequest { return nil } + +func (c *refreshApiClient) SetAfterFunction(callback plugin.ApiClientAfterResponse) {} + +func (c *refreshApiClient) GetAfterFunction() plugin.ApiClientAfterResponse { return nil } + +func (c *refreshApiClient) Get(path string, query url.Values, headers http.Header) (*http.Response, errors.Error) { + return c.do(http.MethodGet, path, query, nil, headers) +} + +func (c *refreshApiClient) Post(path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error) { + return c.do(http.MethodPost, path, query, body, headers) +} + +func (c *refreshApiClient) do(method, path string, query url.Values, body interface{}, headers http.Header) (*http.Response, errors.Error) { + uri, err := api.GetURIStringPointer(c.endpoint, path, query) + if err != nil { + return nil, err + } + + var reqBody *bytes.Reader + if body != nil { + payload, err := json.Marshal(body) + if err != nil { + return nil, errors.Convert(err) + } + reqBody = bytes.NewReader(payload) + } else { + reqBody = bytes.NewReader(nil) + } + + ctx, cancel := context.WithTimeout(context.Background(), c.timeout) + defer cancel() + + req, err := errors.Convert01(http.NewRequestWithContext(ctx, method, *uri, reqBody)) + if err != nil { + return nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + for name, values := range headers { + for _, value := range values { + req.Header.Add(name, value) + } + } + + res, err := errors.Convert01(c.client.Do(req)) + if err != nil { + return nil, err + } + return res, nil +} diff --git a/backend/plugins/github/token/refresh_api_client_test.go b/backend/plugins/github/token/refresh_api_client_test.go new file mode 100644 index 00000000000..60cdbba2f40 --- /dev/null +++ b/backend/plugins/github/token/refresh_api_client_test.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package token + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRefreshApiClientPost(t *testing.T) { + var receivedMethod string + var receivedPath string + var receivedAuth string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedMethod = r.Method + receivedPath = r.URL.Path + receivedAuth = r.Header.Get("Authorization") + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"token":"ghs_test123","expires_at":"2026-03-02T12:00:00Z"}`)) + })) + defer server.Close() + + client := newRefreshApiClientWithTransport(server.URL, server.Client().Transport) + + headers := http.Header{ + "Authorization": []string{"Bearer jwt_token_here"}, + } + resp, err := client.Post("/app/installations/123/access_tokens", nil, nil, headers) + + assert.Nil(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + assert.Equal(t, "POST", receivedMethod) + assert.Equal(t, "/app/installations/123/access_tokens", receivedPath) + assert.Equal(t, "Bearer jwt_token_here", receivedAuth) + + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + assert.Contains(t, string(body), "ghs_test123") +} + +func TestRefreshApiClientGet(t *testing.T) { + var receivedMethod string + var receivedQuery string + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedMethod = r.Method + receivedQuery = r.URL.Query().Get("page") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + client := newRefreshApiClientWithTransport(server.URL, server.Client().Transport) + + resp, err := client.Get("/test", map[string][]string{"page": {"2"}}, nil) + assert.Nil(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "GET", receivedMethod) + assert.Equal(t, "2", receivedQuery) + resp.Body.Close() +} + +func TestRefreshApiClientTimeout(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Sleep longer than the timeout + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + // Use a client without its own timeout so the context timeout is the only constraint + client := newRefreshApiClientWithTransport(server.URL, http.DefaultTransport) + // Override the timeout to something short for the test + client.(*refreshApiClient).timeout = 100 * time.Millisecond + + resp, err := client.Post("/slow", nil, nil, nil) + assert.NotNil(t, err) + if resp != nil { + resp.Body.Close() + } + // Verify the error is a deadline/context error + assert.True(t, strings.Contains(err.Error(), "deadline") || strings.Contains(err.Error(), "context"), + "expected deadline/context error, got: %s", err.Error()) +} diff --git a/backend/plugins/github/token/round_tripper.go b/backend/plugins/github/token/round_tripper.go index 45ba3e9a7b6..8f05f838d0d 100644 --- a/backend/plugins/github/token/round_tripper.go +++ b/backend/plugins/github/token/round_tripper.go @@ -19,6 +19,8 @@ package token import ( "net/http" + "strings" + "sync/atomic" ) // RefreshRoundTripper is an HTTP transport middleware that automatically manages OAuth token refreshes. @@ -26,6 +28,11 @@ import ( // On 401's the round tripper will: // - Force a refresh of the OAuth token via the TokenProvider // - Retry the original request with the new token +// +// Note: When active, the RefreshRoundTripper overwrites the Authorization header +// on every request, superseding any header previously set by SetupAuthentication. +// This is intentional — for connections using token refresh (OAuth or GitHub App), +// the round tripper is the single source of truth for the current bearer token. type RefreshRoundTripper struct { base http.RoundTripper tokenProvider *TokenProvider @@ -88,3 +95,37 @@ func (rt *RefreshRoundTripper) roundTripWithRetry(req *http.Request, refreshAtte return resp, nil } + +// StaticRoundTripper is an HTTP transport that injects a fixed bearer token. +// Unlike RefreshRoundTripper, it does NOT attempt refresh or retries. +type StaticRoundTripper struct { + base http.RoundTripper + tokens []string + idx atomic.Uint64 +} + +func NewStaticRoundTripper(base http.RoundTripper, rawToken string) *StaticRoundTripper { + if base == nil { + base = http.DefaultTransport + } + parts := strings.Split(rawToken, ",") + tokens := make([]string, 0, len(parts)) + for _, t := range parts { + if t = strings.TrimSpace(t); t != "" { + tokens = append(tokens, t) + } + } + if len(tokens) == 0 { + tokens = []string{rawToken} + } + return &StaticRoundTripper{base: base, tokens: tokens} +} + +func (rt *StaticRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // always overrides headers put by SetupAuthentication, to make sure the token is always injected + // Add(1)-1 yields a 0-based sequence (0, 1, 2, ...) so rotation starts at tokens[0]. + tok := rt.tokens[(rt.idx.Add(1)-1)%uint64(len(rt.tokens))] + reqClone := req.Clone(req.Context()) + reqClone.Header.Set("Authorization", "Bearer "+tok) + return rt.base.RoundTrip(reqClone) +} diff --git a/backend/plugins/github/token/round_tripper_test.go b/backend/plugins/github/token/round_tripper_test.go index 460026d98a6..2748d59e118 100644 --- a/backend/plugins/github/token/round_tripper_test.go +++ b/backend/plugins/github/token/round_tripper_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/impls/logruslog" "github.com/apache/incubator-devlake/plugins/github/models" @@ -32,15 +33,41 @@ import ( "github.com/stretchr/testify/mock" ) -// timePtr is a helper function to create a pointer to a time.Time value -func timePtrRT(t time.Time) *time.Time { - return &t -} +// testRSAKey is a throwaway 2048-bit RSA private key used only in tests. +// It is NOT a real credential. +const testRSAKey = `-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEA1xJuX407giVTO/FY2pbp6bdB/XxaiPqAuvWIcEqabzq+d3ft +O7fGtbXSQrCdtxEQt5dHFKdJofHcGlKPDnq1BNjWM3/xLWsQWQPSwUZ9H1qy/nDI +GX+ciXmP8hbzoe5B1OXidAdrJUGWH3ox8Yp8OVd/yK9p34teCbzPnqVEc9tkgUT1 +94gHKLmvP28VefybFyGbYB3ujVIuA8Z26c4gQsyFzR2v1fVeDIu1e1afyH5WTFgr +EWaztOo6pI5stzH00U7fNMzULBuYQ+ufQ4iQ7Ewt7fK5yyQNkx3pX0o1OQ/aYxQr +hyBrHakxHBe78eq8rSR2KwSr5nDuYzIAOUhtnwIDAQABAoH/EknhZ6EkvRPR7G5e +bq4/NduuSriDcITwbWuuuFPufzgdIZVzlu8xHfPOT9152qE2aD7XahGUk85fwbXh +EeP64x/cA3FnmrAxPYfuUFnB0+i7KHGpzW2Wa/LqXlz3vS/UQePoDwl+xDYeUt2u +xALHoSxZdlTWv6HLhLmw7ge9QJLc/xQO9dC678c4Y4JTCRrEvhE+eZiDEXb6HL2D +uNMEwFqMLTxOurYKXE+iyzKg0e4D6oDkw6BC+vOUBnuH+9iKV6wnal24E1WnyhCa +vy7iHBoc8npeKYAHjU5wKDQiWPMT9DjYBRucHvSyEYTeS5eyQHed42PGv+Ss2IL4 +w0JZAoGBAPjxiEJwck29NyfElw5+/P0IKIE8BkuIOSC8JoGfo+JpKYWFIXaLRbK8 +qvOiLDotHgC5IxFZyN8pejVe7Zvif5PepR9qMcV3e6Uz2nmJqtp5gFlzjNUSzbeM +J6gqkejtwn7wJY3dwZhfbTTDDY7cZ9f3Cydarx+iu07unGCRT9DDAoGBAN0rG/F9 +tsOSwrgsnPwNeVPfJn2Zw6mvb1xuOoJl11tDnpS6tH50FBJ/TkERZak9+VZ761pq +CvuMjLtePgC31rsAcUEEt1OwYCxne8gNxuzl2mWKD98ActbFf96VM4Q68Jvntxl1 +eOMMnSA+/yjvhpxdVabm3180mRu9eBv6SaH1AoGBAMpwL6xHoMwS6L1QIr7JCZYC +gl3FoCDgIAS8vFuApFbDyd4oSvQJgZ49yo7g/DI66kEQTLIZXz4KjrTEA1lWsQRg +c8q+Isc/yK6pIirfhq6vS25yhr3m0p9GPCGGrKzMW/O5+fAJuxrbzwSu8WGRXmjD +HrDcD7kcLlGbvFLTGCLdAoGBALNID7W5Z16v6AItv++d6Hzxhh0IeRBi8s2lWO59 +KY6EiNcdZdSfuemoosGiHZuMbkMJ3qWDEnYI38e+xFoGrB0YZbYD4awIbF1yYWew +q1E7ncbznJvznCO3I0lF/uWwdXyb39PWYvECN5h9GI+RYrf7/MN3oRhm5boT43oi +cG/FAoGBALD1zSG9qrysLg8fw4dc4cs6dAHZhAszh47zw2WiMmUVWQZCbB+uhs44 +qGCAaer5KdnTqy9NdwW6rlcr4Y8jUFnMlMFA5HdmHkIfgTi+zi4Qf1mb9yzpJqnU +jKflsh1Lyyqv2KsoIMz4vjew+lCVn80FZaEEQ1q9tAkyu5m53w65 +-----END RSA PRIVATE KEY-----` func TestRoundTripper401Refresh(t *testing.T) { mockRT := new(MockRoundTripper) client := &http.Client{Transport: mockRT} + expiry := time.Now().Add(10 * time.Minute) // Not expired conn := &models.GithubConnection{ GithubConn: models.GithubConn{ RefreshToken: "refresh_token", @@ -49,7 +76,7 @@ func TestRoundTripper401Refresh(t *testing.T) { Token: "old_token", }, }, - TokenExpiresAt: timePtrRT(time.Now().Add(10 * time.Minute)), // Not expired + TokenExpiresAt: &expiry, GithubAppKey: models.GithubAppKey{ AppKey: api.AppKey{ AppId: "123", @@ -60,7 +87,7 @@ func TestRoundTripper401Refresh(t *testing.T) { } logger, _ := logruslog.NewDefaultLogger(logrus.New()) - tp := NewTokenProvider(conn, nil, client, logger) + tp := NewTokenProvider(conn, nil, client, logger, "") rt := NewRefreshRoundTripper(mockRT, tp) // Request @@ -104,3 +131,163 @@ func TestRoundTripper401Refresh(t *testing.T) { mockRT.AssertExpectations(t) } + +func TestRoundTripper401WithAppKeyRefresh(t *testing.T) { + mockRT := new(MockRoundTripper) + + expiry := time.Now().Add(10 * time.Minute) // Not expired (proactive refresh won't trigger) + conn := &models.GithubConnection{ + GithubConn: models.GithubConn{ + GithubAccessToken: models.GithubAccessToken{ + AccessToken: api.AccessToken{ + Token: "old_app_token", + }, + }, + TokenExpiresAt: &expiry, + }, + } + // Use tokens slice so GetToken returns the current token + conn.UpdateToken("old_app_token", "", &expiry, nil) + + // refreshFn simulates minting a new installation token + refreshCalled := 0 + tp := &TokenProvider{ + conn: conn, + refreshFn: func(tp *TokenProvider) errors.Error { + refreshCalled++ + newExpiry := time.Now().Add(1 * time.Hour) + tp.conn.UpdateToken("new_app_token", "", &newExpiry, nil) + return nil + }, + } + + rt := NewRefreshRoundTripper(mockRT, tp) + + req, _ := http.NewRequest("GET", "https://api.github.com/repos/test/test", nil) + + // 1. First call returns 401 + resp401 := &http.Response{ + StatusCode: 401, + Body: io.NopCloser(bytes.NewBufferString("Bad credentials")), + } + mockRT.On("RoundTrip", mock.MatchedBy(func(r *http.Request) bool { + return r.Header.Get("Authorization") == "Bearer old_app_token" + })).Return(resp401, nil).Once() + + // 2. Retry call with new token (after refreshFn runs) + resp200 := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`{"full_name":"test/test"}`)), + } + mockRT.On("RoundTrip", mock.MatchedBy(func(r *http.Request) bool { + return r.Header.Get("Authorization") == "Bearer new_app_token" + })).Return(resp200, nil).Once() + + // Execute + resp, err := rt.RoundTrip(req) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, 1, refreshCalled, "refreshFn should have been called exactly once") + + body, _ := io.ReadAll(resp.Body) + assert.Equal(t, `{"full_name":"test/test"}`, string(body)) + + mockRT.AssertExpectations(t) +} + +// TestProactiveRefreshNoDeadlock verifies that when the RefreshRoundTripper wraps +// the same http.Client's transport, a proactive token refresh does not deadlock. +// This reproduces the real-world scenario: GetToken() holds the mutex, calls +// refreshGitHubAppInstallationToken, which makes an HTTP request. If that request +// goes through the RefreshRoundTripper (re-entering GetToken), it would deadlock. +// The fix is that the refresh uses baseTransport directly. +func TestProactiveRefreshNoDeadlock(t *testing.T) { + // Set up a mock transport that will serve both: + // 1. The installation token refresh POST + // 2. The actual API GET after refresh + mockRT := new(MockRoundTripper) + client := &http.Client{Transport: mockRT} + + // Token is expired — proactive refresh WILL trigger on GetToken() + expired := time.Now().Add(-1 * time.Minute) + conn := &models.GithubConnection{ + GithubConn: models.GithubConn{ + RestConnection: api.RestConnection{ + Endpoint: "https://api.github.com/", + }, + MultiAuth: api.MultiAuth{ + AuthMethod: models.AppKey, + }, + GithubAccessToken: models.GithubAccessToken{ + AccessToken: api.AccessToken{ + Token: "expired_ghs_token", + }, + }, + GithubAppKey: models.GithubAppKey{ + AppKey: api.AppKey{ + AppId: "12345", + SecretKey: testRSAKey, + }, + InstallationID: 99999, + }, + TokenExpiresAt: &expired, + }, + } + conn.UpdateToken("expired_ghs_token", "", &expired, nil) + + // Create the TokenProvider with baseTransport = mockRT (the unwrapped transport). + // This is what NewAppInstallationTokenProvider does: it captures client.Transport + // BEFORE the caller wraps it with RefreshRoundTripper. + tp := NewAppInstallationTokenProvider(conn, nil, client, nil, "") + + // Now wrap the client's transport with RefreshRoundTripper (simulating what + // CreateApiClient does). After this, client.Transport = RefreshRoundTripper, + // but tp.baseTransport still points to mockRT. + rt := NewRefreshRoundTripper(mockRT, tp) + client.Transport = rt + + // Mock: installation token refresh POST (goes through baseTransport, not RT) + newExpiry := time.Now().Add(1 * time.Hour) + installTokenBody := `{"token":"new_ghs_token","expires_at":"` + newExpiry.Format(time.RFC3339) + `"}` + mockRT.On("RoundTrip", mock.MatchedBy(func(r *http.Request) bool { + return r.Method == "POST" && r.URL.Path == "/app/installations/99999/access_tokens" + })).Return(&http.Response{ + StatusCode: 201, + Body: io.NopCloser(bytes.NewBufferString(installTokenBody)), + }, nil).Once() + + // Mock: the actual API request (goes through RefreshRoundTripper → GetToken → mockRT) + mockRT.On("RoundTrip", mock.MatchedBy(func(r *http.Request) bool { + return r.Method == "GET" && r.URL.Path == "/repos/test/test" + })).Return(&http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBufferString(`{"full_name":"test/test"}`)), + }, nil).Once() + + // Execute through the RefreshRoundTripper with a timeout to detect deadlocks. + done := make(chan struct{}) + var resp *http.Response + var reqErr error + go func() { + req, _ := http.NewRequest("GET", "https://api.github.com/repos/test/test", nil) + resp, reqErr = rt.RoundTrip(req) + close(done) + }() + + select { + case <-done: + // Success — no deadlock + case <-time.After(5 * time.Second): + t.Fatal("DEADLOCK: RoundTrip did not complete within 5 seconds — " + + "the refresh call is likely going through RefreshRoundTripper instead of baseTransport") + } + + assert.NoError(t, reqErr) + assert.Equal(t, 200, resp.StatusCode) + assert.Equal(t, "new_ghs_token", conn.Token, "token should have been refreshed") + + body, _ := io.ReadAll(resp.Body) + assert.Equal(t, `{"full_name":"test/test"}`, string(body)) + + mockRT.AssertExpectations(t) +} diff --git a/backend/plugins/github/token/token_provider.go b/backend/plugins/github/token/token_provider.go index 15aa09ca80e..6af31aaf5af 100644 --- a/backend/plugins/github/token/token_provider.go +++ b/backend/plugins/github/token/token_provider.go @@ -40,23 +40,57 @@ const ( ) type TokenProvider struct { - conn *models.GithubConnection - dal dal.Dal - httpClient *http.Client - logger log.Logger - mu sync.Mutex - refreshURL string + conn *models.GithubConnection + dal dal.Dal + encryptionSecret string + httpClient *http.Client + baseTransport http.RoundTripper // original transport, before RefreshRoundTripper wrapping + logger log.Logger + mu sync.Mutex + refreshURL string + refreshFn func(*TokenProvider) errors.Error } // NewTokenProvider creates a TokenProvider for the given GitHub connection using // the provided DAL, HTTP client, and logger, and returns a pointer to it. -func NewTokenProvider(conn *models.GithubConnection, d dal.Dal, client *http.Client, logger log.Logger) *TokenProvider { +func NewTokenProvider(conn *models.GithubConnection, d dal.Dal, client *http.Client, logger log.Logger, encryptionSecret string) *TokenProvider { return &TokenProvider{ - conn: conn, - dal: d, - httpClient: client, - logger: logger, - refreshURL: "https://github.com/login/oauth/access_token", + conn: conn, + dal: d, + encryptionSecret: encryptionSecret, + httpClient: client, + logger: logger, + refreshURL: "https://github.com/login/oauth/access_token", + } +} + +// NewAppInstallationTokenProvider creates a TokenProvider that refreshes GitHub App installation tokens. +// IMPORTANT: Call this BEFORE wrapping the client's transport with RefreshRoundTripper, +// so that baseTransport captures the unwrapped transport and refresh calls don't deadlock. +func NewAppInstallationTokenProvider(conn *models.GithubConnection, d dal.Dal, client *http.Client, logger log.Logger, encryptionSecret string) *TokenProvider { + if logger != nil { + expiresStr := "unknown" + if conn.TokenExpiresAt != nil { + expiresStr = conn.TokenExpiresAt.Format(time.RFC3339) + } + logger.Info("Created AppInstallation token provider for connection %d (installation %d, token expires at %s)", + conn.ID, conn.InstallationID, expiresStr) + } + // Capture the transport now, before the caller wraps it with RefreshRoundTripper. + // This avoids a deadlock: refresh calls must bypass the RefreshRoundTripper that + // holds the TokenProvider mutex during GetToken(). + baseTransport := client.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + return &TokenProvider{ + conn: conn, + dal: d, + encryptionSecret: encryptionSecret, + httpClient: client, + baseTransport: baseTransport, + logger: logger, + refreshFn: refreshGitHubAppInstallationToken, } } @@ -65,6 +99,14 @@ func (tp *TokenProvider) GetToken() (string, errors.Error) { defer tp.mu.Unlock() if tp.needsRefresh() { + if tp.logger != nil { + expiresStr := "unknown" + if tp.conn.TokenExpiresAt != nil { + expiresStr = tp.conn.TokenExpiresAt.Format(time.RFC3339) + } + tp.logger.Info("Proactive token refresh triggered for connection %d (token expires at %s)", + tp.conn.ID, expiresStr) + } if err := tp.refreshToken(); err != nil { return "", err } @@ -73,10 +115,6 @@ func (tp *TokenProvider) GetToken() (string, errors.Error) { } func (tp *TokenProvider) needsRefresh() bool { - if tp.conn.RefreshToken == "" { - return false - } - buffer := DefaultRefreshBuffer if envBuffer := os.Getenv("GITHUB_TOKEN_REFRESH_BUFFER_MINUTES"); envBuffer != "" { if val, err := strconv.Atoi(envBuffer); err == nil { @@ -84,6 +122,16 @@ func (tp *TokenProvider) needsRefresh() bool { } } + if tp.refreshFn != nil { + if tp.conn.TokenExpiresAt == nil { + return false + } + return time.Now().Add(buffer).After(*tp.conn.TokenExpiresAt) + } + + if tp.conn.RefreshToken == "" { + return false + } if tp.conn.TokenExpiresAt == nil { return false } @@ -91,6 +139,9 @@ func (tp *TokenProvider) needsRefresh() bool { } func (tp *TokenProvider) refreshToken() errors.Error { + if tp.refreshFn != nil { + return tp.refreshFn(tp) + } tp.logger.Info("Refreshing GitHub token for connection %d", tp.conn.ID) data := map[string]string{ @@ -148,21 +199,23 @@ func (tp *TokenProvider) refreshToken() errors.Error { return errors.Default.New(fmt.Sprintf("empty access token returned; response body: %s", bodyStr)) } + tokenExpiredAt := time.Now().Add(time.Duration(result.ExpiresIn) * time.Second) + refreshTokenExpiredAt := time.Now().Add(time.Duration(result.RefreshTokenExpiresIn) * time.Second) + tp.conn.UpdateToken( result.AccessToken, result.RefreshToken, - time.Now().Add(time.Duration(result.ExpiresIn)*time.Second), - time.Now().Add(time.Duration(result.RefreshTokenExpiresIn)*time.Second), + &tokenExpiredAt, + &refreshTokenExpiredAt, ) if tp.dal != nil { - err := tp.dal.UpdateColumns(tp.conn, []dal.DalSet{ - {ColumnName: "token", Value: tp.conn.Token}, - {ColumnName: "refresh_token", Value: tp.conn.RefreshToken}, - {ColumnName: "token_expires_at", Value: tp.conn.TokenExpiresAt}, - {ColumnName: "refresh_token_expires_at", Value: tp.conn.RefreshTokenExpiresAt}, - }) - if err != nil { + // Manually encrypt and use UpdateColumns to persist only the token-related + // columns. We cannot use dal.Update (GORM Save) because it writes ALL fields + // including refresh_token_expires_at which may have Go zero time that MySQL + // rejects. We cannot use UpdateColumns with plaintext because it bypasses the + // GORM encdec serializer. So we encrypt manually and write the ciphertext. + if err := PersistEncryptedTokenColumns(tp.dal, tp.conn, tp.encryptionSecret, tp.logger, true); err != nil { tp.logger.Warn(err, "failed to persist refreshed token") } } @@ -181,8 +234,14 @@ func (tp *TokenProvider) ForceRefresh(oldToken string) errors.Error { // If the token has changed since the request was made, it means another thread // has already refreshed it. if tp.conn.Token != oldToken { + if tp.logger != nil { + tp.logger.Info("Skipping reactive token refresh for connection %d — token already changed by another goroutine", tp.conn.ID) + } return nil } + if tp.logger != nil { + tp.logger.Info("Reactive token refresh triggered for connection %d (received 401)", tp.conn.ID) + } return tp.refreshToken() } diff --git a/backend/plugins/github/token/token_provider_test.go b/backend/plugins/github/token/token_provider_test.go index fa0809e7816..a754ad63ce3 100644 --- a/backend/plugins/github/token/token_provider_test.go +++ b/backend/plugins/github/token/token_provider_test.go @@ -29,7 +29,6 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/impls/logruslog" - mockdal "github.com/apache/incubator-devlake/mocks/core/dal" "github.com/apache/incubator-devlake/plugins/github/models" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -45,11 +44,6 @@ func (m *MockRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) return args.Get(0).(*http.Response), args.Error(1) } -// timePtr is a helper function to create a pointer to a time.Time value -func timePtr(t time.Time) *time.Time { - return &t -} - func TestNeedsRefresh(t *testing.T) { tp := &TokenProvider{ conn: &models.GithubConnection{ @@ -60,15 +54,18 @@ func TestNeedsRefresh(t *testing.T) { } // Not expired, outside buffer - tp.conn.TokenExpiresAt = timePtr(time.Now().Add(10 * time.Minute)) + expiry1 := time.Now().Add(10 * time.Minute) + tp.conn.TokenExpiresAt = &expiry1 assert.False(t, tp.needsRefresh()) // Inside buffer - tp.conn.TokenExpiresAt = timePtr(time.Now().Add(1 * time.Minute)) + expiry2 := time.Now().Add(1 * time.Minute) + tp.conn.TokenExpiresAt = &expiry2 assert.True(t, tp.needsRefresh()) // Expired - tp.conn.TokenExpiresAt = timePtr(time.Now().Add(-1 * time.Minute)) + expiry3 := time.Now().Add(-1 * time.Minute) + tp.conn.TokenExpiresAt = &expiry3 assert.True(t, tp.needsRefresh()) // No refresh token @@ -80,10 +77,11 @@ func TestTokenProviderConcurrency(t *testing.T) { mockRT := new(MockRoundTripper) client := &http.Client{Transport: mockRT} + expired := time.Now().Add(-1 * time.Minute) // Expired conn := &models.GithubConnection{ GithubConn: models.GithubConn{ RefreshToken: "refresh_token", - TokenExpiresAt: timePtr(time.Now().Add(-1 * time.Minute)), // Expired + TokenExpiresAt: &expired, GithubAppKey: models.GithubAppKey{ AppKey: api.AppKey{ AppId: "123", @@ -94,7 +92,7 @@ func TestTokenProviderConcurrency(t *testing.T) { } logger, _ := logruslog.NewDefaultLogger(logrus.New()) - tp := NewTokenProvider(conn, nil, client, logger) + tp := NewTokenProvider(conn, nil, client, logger, "") // Mock response for refresh respBody := `{"access_token":"new_token","refresh_token":"new_refresh_token","expires_in":3600,"refresh_token_expires_in":3600}` @@ -134,52 +132,118 @@ func TestConfigurableBuffer(t *testing.T) { } // 9 minutes remaining (inside 10m buffer) - tp.conn.TokenExpiresAt = timePtr(time.Now().Add(9 * time.Minute)) + expiry9 := time.Now().Add(9 * time.Minute) + tp.conn.TokenExpiresAt = &expiry9 assert.True(t, tp.needsRefresh()) // 11 minutes remaining (outside 10m buffer) - tp.conn.TokenExpiresAt = timePtr(time.Now().Add(11 * time.Minute)) + expiry11 := time.Now().Add(11 * time.Minute) + tp.conn.TokenExpiresAt = &expiry11 assert.False(t, tp.needsRefresh()) } -func TestPersistenceFailure(t *testing.T) { - mockRT := new(MockRoundTripper) - client := &http.Client{Transport: mockRT} - mockDal := new(mockdal.Dal) +// fakeRefreshFn returns a refreshFn that updates the connection token to newToken +// and increments the call counter pointed to by count. +func fakeRefreshFn(newToken string, count *int) func(*TokenProvider) errors.Error { + return func(tp *TokenProvider) errors.Error { + *count++ + newExpiry := time.Now().Add(1 * time.Hour) + tp.conn.UpdateToken(newToken, "", &newExpiry, nil) + return nil + } +} + +func TestNeedsRefreshWithRefreshFn(t *testing.T) { + callCount := 0 + tp := &TokenProvider{ + conn: &models.GithubConnection{}, + refreshFn: fakeRefreshFn("unused", &callCount), + } + + // Token not expired — outside default 5m buffer + expiry1 := time.Now().Add(10 * time.Minute) + tp.conn.TokenExpiresAt = &expiry1 + assert.False(t, tp.needsRefresh(), "should not refresh when token is 10m from expiry") + + // Token inside 5m buffer + expiry2 := time.Now().Add(2 * time.Minute) + tp.conn.TokenExpiresAt = &expiry2 + assert.True(t, tp.needsRefresh(), "should refresh when token is 2m from expiry") + // Token already expired + expiry3 := time.Now().Add(-1 * time.Minute) + tp.conn.TokenExpiresAt = &expiry3 + assert.True(t, tp.needsRefresh(), "should refresh when token is expired") + + // TokenExpiresAt is nil — can't determine expiry, don't refresh (401 fallback covers this) + tp.conn.TokenExpiresAt = nil + assert.False(t, tp.needsRefresh(), "should not refresh when TokenExpiresAt is nil") + + // Provider with neither refreshFn nor RefreshToken — should never refresh + tp2 := &TokenProvider{ + conn: &models.GithubConnection{}, + } + expiry4 := time.Now().Add(-1 * time.Minute) + tp2.conn.TokenExpiresAt = &expiry4 + assert.False(t, tp2.needsRefresh(), "should not refresh without refreshFn or RefreshToken") +} + +func TestAppKeyGetTokenTriggersRefresh(t *testing.T) { + callCount := 0 + expired := time.Now().Add(-1 * time.Minute) conn := &models.GithubConnection{ GithubConn: models.GithubConn{ - RefreshToken: "refresh_token", GithubAccessToken: models.GithubAccessToken{ AccessToken: api.AccessToken{ - Token: "old_token", + Token: "expired_token", }, }, - GithubAppKey: models.GithubAppKey{ - AppKey: api.AppKey{ - AppId: "123", - SecretKey: "secret", + TokenExpiresAt: &expired, + }, + } + + tp := &TokenProvider{ + conn: conn, + refreshFn: fakeRefreshFn("refreshed_app_token", &callCount), + } + + token, err := tp.GetToken() + assert.NoError(t, err) + assert.Equal(t, "refreshed_app_token", token) + assert.Equal(t, 1, callCount, "refreshFn should have been called exactly once") + + // Second call — token is now fresh, should not trigger refresh + token2, err := tp.GetToken() + assert.NoError(t, err) + assert.Equal(t, "refreshed_app_token", token2) + assert.Equal(t, 1, callCount, "refreshFn should not be called again for a fresh token") +} + +func TestAppKeyForceRefresh(t *testing.T) { + callCount := 0 + conn := &models.GithubConnection{ + GithubConn: models.GithubConn{ + GithubAccessToken: models.GithubAccessToken{ + AccessToken: api.AccessToken{ + Token: "old_app_token", }, }, }, } - logger, _ := logruslog.NewDefaultLogger(logrus.New()) - tp := NewTokenProvider(conn, mockDal, client, logger) - - // Mock response for refresh - respBody := `{"access_token":"new_token","refresh_token":"new_refresh_token","expires_in":3600,"refresh_token_expires_in":3600}` - resp := &http.Response{ - StatusCode: 200, - Body: io.NopCloser(bytes.NewBufferString(respBody)), + tp := &TokenProvider{ + conn: conn, + refreshFn: fakeRefreshFn("new_app_token", &callCount), } - mockRT.On("RoundTrip", mock.Anything).Return(resp, nil).Once() - // Mock DAL failure - mockDal.On("UpdateColumns", mock.Anything, mock.Anything, mock.AnythingOfType("[]dal.Clause")).Return(errors.Default.New("db error")) - err := tp.ForceRefresh("old_token") - assert.NoError(t, err) // Should not return error even if persistence fails + // ForceRefresh with matching old token — should trigger refresh + err := tp.ForceRefresh("old_app_token") + assert.NoError(t, err) + assert.Equal(t, 1, callCount) + assert.Equal(t, "new_app_token", conn.Token) - mockRT.AssertExpectations(t) - mockDal.AssertExpectations(t) + // ForceRefresh with stale old token — token has already changed, should be a no-op + err = tp.ForceRefresh("old_app_token") + assert.NoError(t, err) + assert.Equal(t, 1, callCount, "should not refresh when token has already changed") } diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go index 31fe0a9b75e..f56c77644bf 100644 --- a/backend/plugins/github_graphql/impl/impl.go +++ b/backend/plugins/github_graphql/impl/impl.go @@ -20,10 +20,7 @@ package impl import ( "context" "fmt" - "net/http" - "net/url" "reflect" - "strings" "time" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -38,8 +35,7 @@ import ( githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" "github.com/apache/incubator-devlake/plugins/github_graphql/model/migrationscripts" "github.com/apache/incubator-devlake/plugins/github_graphql/tasks" - "github.com/merico-dev/graphql" - "golang.org/x/oauth2" + "github.com/merico-ai/graphql" ) // make sure interface is implemented @@ -180,46 +176,10 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return nil, err } - tokens := strings.Split(connection.Token, ",") - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: tokens[0]}, - ) - oauthContext := taskCtx.GetContext() - proxy := connection.GetProxy() - if proxy != "" { - pu, err := url.Parse(proxy) - if err != nil { - return nil, errors.Convert(err) - } - if pu.Scheme == "http" || pu.Scheme == "socks5" { - proxyClient := &http.Client{ - Transport: &http.Transport{Proxy: http.ProxyURL(pu)}, - } - oauthContext = context.WithValue( - taskCtx.GetContext(), - oauth2.HTTPClient, - proxyClient, - ) - logger.Debug("Proxy set in oauthContext to %s", proxy) - } else { - return nil, errors.BadInput.New("Unsupported scheme set in proxy") - } - } - - httpClient := oauth2.NewClient(oauthContext, src) - endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) - if err != nil { - return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) - } - - // github.com and github enterprise have different graphql endpoints - endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql - if endpoint.Hostname() != "api.github.com" { - // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql - endpoint.Path = "/api/graphql" - } - client := graphql.NewClient(endpoint.String(), httpClient) - graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + graphqlClient, err := tasks.CreateGraphqlClient( + taskCtx, + connection, + apiClient.ApiClient.GetClient(), func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) { var query GraphQueryRateLimit dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil)) @@ -230,8 +190,7 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`) } if query.RateLimit == nil { - logger.Info(`github graphql rate limit are disabled, fallback to 5000req/hour`) - return 5000, nil, nil + return 0, nil, errors.Default.New("rate limit unavailable") } logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`, query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt) diff --git a/backend/plugins/github_graphql/tasks/account_collector.go b/backend/plugins/github_graphql/tasks/account_collector.go index 34e4faac109..a75013d0d1d 100644 --- a/backend/plugins/github_graphql/tasks/account_collector.go +++ b/backend/plugins/github_graphql/tasks/account_collector.go @@ -27,7 +27,7 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) const RAW_ACCOUNTS_TABLE = "github_graphql_accounts" diff --git a/backend/plugins/github_graphql/tasks/deployment_collector.go b/backend/plugins/github_graphql/tasks/deployment_collector.go index af97330e062..e224faf27de 100644 --- a/backend/plugins/github_graphql/tasks/deployment_collector.go +++ b/backend/plugins/github_graphql/tasks/deployment_collector.go @@ -26,7 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) var _ plugin.SubTaskEntryPoint = CollectDeployments diff --git a/backend/plugins/github_graphql/tasks/graphql_client.go b/backend/plugins/github_graphql/tasks/graphql_client.go new file mode 100644 index 00000000000..9c248e15cb6 --- /dev/null +++ b/backend/plugins/github_graphql/tasks/graphql_client.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/merico-ai/graphql" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +func CreateGraphqlClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + httpClient *http.Client, + getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), +) (*helper.GraphqlAsyncClient, errors.Error) { + // Build endpoint + endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) + } + // github.com and github enterprise have different graphql endpoints + if endpoint.Hostname() == "api.github.com" { + // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/graphql" + } else { + // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/api/graphql" + } + + gqlClient := graphql.NewClient(endpoint.String(), httpClient) + + return helper.CreateAsyncGraphqlClient( + taskCtx, + gqlClient, + taskCtx.GetLogger(), + getRateRemaining, + // GitHub GraphQL default fallback aligns with GitHub's standard rate limit (~5000) + helper.WithFallbackRateLimit(5000), + ) +} diff --git a/backend/plugins/github_graphql/tasks/issue_collector.go b/backend/plugins/github_graphql/tasks/issue_collector.go index fb5aa13e24c..36c33584c8a 100644 --- a/backend/plugins/github_graphql/tasks/issue_collector.go +++ b/backend/plugins/github_graphql/tasks/issue_collector.go @@ -34,7 +34,7 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) const RAW_ISSUES_TABLE = "github_graphql_issues" diff --git a/backend/plugins/github_graphql/tasks/job_collector.go b/backend/plugins/github_graphql/tasks/job_collector.go index f0deb7b41bf..05fd670078b 100644 --- a/backend/plugins/github_graphql/tasks/job_collector.go +++ b/backend/plugins/github_graphql/tasks/job_collector.go @@ -29,7 +29,7 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) const RAW_GRAPHQL_JOBS_TABLE = "github_graphql_jobs" diff --git a/backend/plugins/github_graphql/tasks/pr_collector.go b/backend/plugins/github_graphql/tasks/pr_collector.go index b3d46246755..669839385d4 100644 --- a/backend/plugins/github_graphql/tasks/pr_collector.go +++ b/backend/plugins/github_graphql/tasks/pr_collector.go @@ -29,7 +29,7 @@ import ( "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) const RAW_PRS_TABLE = "github_graphql_prs" diff --git a/backend/plugins/github_graphql/tasks/release_collector.go b/backend/plugins/github_graphql/tasks/release_collector.go index 83581c4c1dd..1f9f86c5de3 100644 --- a/backend/plugins/github_graphql/tasks/release_collector.go +++ b/backend/plugins/github_graphql/tasks/release_collector.go @@ -27,7 +27,7 @@ import ( "github.com/apache/incubator-devlake/core/utils" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" githubTasks "github.com/apache/incubator-devlake/plugins/github/tasks" - "github.com/merico-dev/graphql" + "github.com/merico-ai/graphql" ) const RAW_RELEASE_TABLE = "github_graphql_release" diff --git a/backend/plugins/gitlab/e2e/job_test.go b/backend/plugins/gitlab/e2e/job_test.go index 1705f59a8c6..64f45824032 100644 --- a/backend/plugins/gitlab/e2e/job_test.go +++ b/backend/plugins/gitlab/e2e/job_test.go @@ -71,7 +71,7 @@ func TestGitlabJobDataFlow(t *testing.T) { ), ) - // verifi when production regex is omitted + // verify when production regex is omitted dataflowTester.FlushTabler(&devops.CICDTask{}) dataflowTester.Subtask(tasks.ConvertJobMeta, taskData) dataflowTester.VerifyTableWithOptions(&devops.CICDTask{}, e2ehelper.TableOptions{ diff --git a/backend/plugins/gitlab/models/migrationscripts/archived/connection.go b/backend/plugins/gitlab/models/migrationscripts/archived/connection.go index f7b7f9f6783..7d717b5ddd7 100644 --- a/backend/plugins/gitlab/models/migrationscripts/archived/connection.go +++ b/backend/plugins/gitlab/models/migrationscripts/archived/connection.go @@ -31,7 +31,7 @@ type RestConnection struct { BaseConnection `mapstructure:",squash"` Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` Proxy string `mapstructure:"proxy" json:"proxy"` - RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimit"` } type BaseConnection struct { diff --git a/backend/plugins/gitlab/tasks/shared.go b/backend/plugins/gitlab/tasks/shared.go index 3c100d7dd2e..6621ba3f2fd 100644 --- a/backend/plugins/gitlab/tasks/shared.go +++ b/backend/plugins/gitlab/tasks/shared.go @@ -187,7 +187,7 @@ func GetMergeRequestsIterator(taskCtx plugin.SubTaskContext, apiCollector *api.S clauses := []dal.Clause{ dal.Select("gmr.gitlab_id, gmr.iid"), dal.From("_tool_gitlab_merge_requests gmr"), - // collect only openning merge request's notes and commits to speed up the process + // collect only opening merge request's notes and commits to speed up the process dal.Where( `gmr.project_id = ? and gmr.connection_id = ?`, data.Options.ProjectId, data.Options.ConnectionId, diff --git a/backend/plugins/icla/api/swagger.go b/backend/plugins/icla/api/swagger.go index 8f9a0f607f5..06db41f1f5d 100644 --- a/backend/plugins/icla/api/swagger.go +++ b/backend/plugins/icla/api/swagger.go @@ -25,7 +25,7 @@ package api // @Router /blueprints/icla/blueprint-setting [post] func _() {} -//nolint:unused,deadcode +//nolint:unused type iclaBlueprintSetting []struct { Version string `json:"version" example:"1.0.0"` Connections []struct { @@ -41,7 +41,7 @@ type iclaBlueprintSetting []struct { // @Router /pipelines/icla/pipeline-plan [post] func _() {} -//nolint:unused,deadcode +//nolint:unused type iclaPipelinePlan [][]struct { Plugin string `json:"plugin" example:"icla"` } diff --git a/backend/plugins/issue_trace/tasks/issue_status_history_convertor.go b/backend/plugins/issue_trace/tasks/issue_status_history_convertor.go index 42bfd0fd328..efe8ddb4f9a 100644 --- a/backend/plugins/issue_trace/tasks/issue_status_history_convertor.go +++ b/backend/plugins/issue_trace/tasks/issue_status_history_convertor.go @@ -219,7 +219,7 @@ func ConvertIssueStatusHistory(taskCtx plugin.SubTaskContext) errors.Error { } } } - logger.Info("issues status history covert successfully") + logger.Info("issues status history converted successfully") return nil } diff --git a/backend/plugins/jenkins/models/build.go b/backend/plugins/jenkins/models/build.go index 203d81d0023..b38aaa15afa 100644 --- a/backend/plugins/jenkins/models/build.go +++ b/backend/plugins/jenkins/models/build.go @@ -35,7 +35,7 @@ type JenkinsBuild struct { Number int64 `gorm:"index"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp Type string `gorm:"index;type:varchar(255)"` Class string `gorm:"index;type:varchar(255)" ` TriggeredBy string `gorm:"type:varchar(255)"` diff --git a/backend/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go b/backend/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go index c0d49a0e913..ec4e3322966 100644 --- a/backend/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go +++ b/backend/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go @@ -40,7 +40,7 @@ type jenkinsBuild20220916Before struct { Number int64 `gorm:"primaryKey"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp Type string `gorm:"index;type:varchar(255)"` Class string `gorm:"index;type:varchar(255)" ` TriggeredBy string `gorm:"type:varchar(255)"` @@ -63,7 +63,7 @@ type jenkinsBuild20220916After struct { Number int64 `gorm:"index"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp Type string `gorm:"index;type:varchar(255)"` Class string `gorm:"index;type:varchar(255)" ` TriggeredBy string `gorm:"type:varchar(255)"` diff --git a/backend/plugins/jenkins/models/migrationscripts/20221131_add_fullName_for_builds.go b/backend/plugins/jenkins/models/migrationscripts/20221131_add_fullName_for_builds.go index a7c0692c25c..2802835e033 100644 --- a/backend/plugins/jenkins/models/migrationscripts/20221131_add_fullName_for_builds.go +++ b/backend/plugins/jenkins/models/migrationscripts/20221131_add_fullName_for_builds.go @@ -48,7 +48,7 @@ type jenkinsBuild20221131Before struct { Number int64 `gorm:"index"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp Type string `gorm:"index;type:varchar(255)"` Class string `gorm:"index;type:varchar(255)" ` TriggeredBy string `gorm:"type:varchar(255)"` @@ -72,7 +72,7 @@ type jenkinsBuild20221131After struct { Number int64 `gorm:"index"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp Type string `gorm:"index;type:varchar(255)"` Class string `gorm:"index;type:varchar(255)" ` TriggeredBy string `gorm:"type:varchar(255)"` diff --git a/backend/plugins/jenkins/models/migrationscripts/archived/build.go b/backend/plugins/jenkins/models/migrationscripts/archived/build.go index 4659e0cc2da..70bc781f62d 100644 --- a/backend/plugins/jenkins/models/migrationscripts/archived/build.go +++ b/backend/plugins/jenkins/models/migrationscripts/archived/build.go @@ -35,7 +35,7 @@ type JenkinsBuild struct { Number int64 `gorm:"primaryKey"` Result string // Result Timestamp int64 // start time - StartTime time.Time // convered by timestamp + StartTime time.Time // converted by timestamp CommitSha string `gorm:"type:varchar(255)"` } diff --git a/backend/plugins/jenkins/models/response.go b/backend/plugins/jenkins/models/response.go index 643bec69136..93eae397bab 100644 --- a/backend/plugins/jenkins/models/response.go +++ b/backend/plugins/jenkins/models/response.go @@ -61,6 +61,11 @@ func (j Job) ToJenkinsJob() *JenkinsJob { } } + primaryView := j.URL + j.Path + j.Class + if len(primaryView) > 255 { + primaryView = primaryView[:255] + } + return &JenkinsJob{ FullName: j.FullName, Name: j.Name, @@ -70,7 +75,7 @@ func (j Job) ToJenkinsJob() *JenkinsJob { Base: j.Base, Url: j.URL, Description: j.Description, - PrimaryView: j.URL + j.Path + j.Class, + PrimaryView: primaryView, } } diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index 126e513e24d..6fb428a9215 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -195,8 +195,8 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int return nil, errors.Default.Wrap(err, "failed to create jira api client") } + var scope *models.JiraBoard if op.BoardId != 0 { - var scope *models.JiraBoard // support v100 & advance mode // If we still cannot find the record in db, we have to request from remote server and save it to db db := taskCtx.GetDal() @@ -249,6 +249,7 @@ func (p Jira) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int Options: &op, ApiClient: jiraApiClient, JiraServerInfo: *info, + Board: scope, } return taskData, nil diff --git a/backend/plugins/jira/models/board.go b/backend/plugins/jira/models/board.go index 267c1cdc0ec..f76af2a5247 100644 --- a/backend/plugins/jira/models/board.go +++ b/backend/plugins/jira/models/board.go @@ -34,6 +34,7 @@ type JiraBoard struct { Self string `json:"self" mapstructure:"self" gorm:"type:varchar(255)"` Type string `json:"type" mapstructure:"type" gorm:"type:varchar(100)"` Jql string `json:"jql" mapstructure:"jql"` + SubQuery string `json:"subQuery" mapstructure:"subQuery"` } func (b JiraBoard) ScopeId() string { diff --git a/backend/plugins/jira/models/migrationscripts/20220716_add_init_tables.go b/backend/plugins/jira/models/migrationscripts/20220716_add_init_tables.go index d27726f76c3..6f161976c0b 100644 --- a/backend/plugins/jira/models/migrationscripts/20220716_add_init_tables.go +++ b/backend/plugins/jira/models/migrationscripts/20220716_add_init_tables.go @@ -45,7 +45,7 @@ type jiraConnection20220716Before struct { StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"` RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"` Proxy string `json:"proxy"` - RateLimit int `comment:"api request rate limt per hour" json:"rateLimit"` + RateLimit int `comment:"api request rate limit per hour" json:"rateLimit"` } type addInitTables20220716 struct{} diff --git a/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go new file mode 100644 index 00000000000..455be7f6df2 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260611_add_sub_query_to_boards.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraBoard20260611 struct { + SubQuery string +} + +func (JiraBoard20260611) TableName() string { + return "_tool_jira_boards" +} + +type addSubQueryToBoards struct{} + +func (script *addSubQueryToBoards) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraBoard20260611{}) +} + +func (*addSubQueryToBoards) Version() uint64 { + return 20260611140000 +} + +func (*addSubQueryToBoards) Name() string { + return "add sub_query to _tool_jira_boards" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go new file mode 100644 index 00000000000..34966a56525 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260702_add_extra_jql_to_scope_config.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type JiraScopeConfig20260702 struct { + ExtraJQL string `gorm:"type:varchar(255)"` +} + +func (JiraScopeConfig20260702) TableName() string { + return "_tool_jira_scope_configs" +} + +type addExtraJQLToScopeConfig struct{} + +func (script *addExtraJQLToScopeConfig) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &JiraScopeConfig20260702{}) +} + +func (*addExtraJQLToScopeConfig) Version() uint64 { + return 20260702000000 +} + +func (*addExtraJQLToScopeConfig) Name() string { + return "add extra_jql to _tool_jira_scope_configs" +} diff --git a/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go b/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go new file mode 100644 index 00000000000..23b37dab081 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20260707_change_fix_versions_to_text.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type jiraIssue20260707 struct { + FixVersions string `gorm:"type:text;column:fix_versions"` +} + +func (jiraIssue20260707) TableName() string { + return "_tool_jira_issues" +} + +type changeFixVersionsToText20260707 struct{} + +func (script *changeFixVersionsToText20260707) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &jiraIssue20260707{}) +} + +func (*changeFixVersionsToText20260707) Version() uint64 { + return 20260707140000 +} + +func (*changeFixVersionsToText20260707) Name() string { + return "change fix_versions type to text in _tool_jira_issues" +} diff --git a/backend/plugins/jira/models/migrationscripts/archived/source.go b/backend/plugins/jira/models/migrationscripts/archived/source.go index 2ceed1613b3..951eba5b598 100644 --- a/backend/plugins/jira/models/migrationscripts/archived/source.go +++ b/backend/plugins/jira/models/migrationscripts/archived/source.go @@ -30,7 +30,7 @@ type JiraSource struct { StoryPointField string `gorm:"type:varchar(50);" json:"storyPointField"` RemotelinkCommitShaPattern string `gorm:"type:varchar(255);comment='golang regexp, the first group will be recognized as commit sha, ref https://github.com/google/re2/wiki/Syntax'" json:"remotelinkCommitShaPattern"` Proxy string `json:"proxy"` - RateLimit int `comment:"api request rate limt per second"` + RateLimit int `comment:"api request rate limit per second"` } type JiraIssueTypeMapping struct { diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 9c334a9ef88..379d54f8964 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -55,5 +55,8 @@ func All() []plugin.MigrationScript { new(flushJiraIssues), new(updateScopeConfig), new(addFixVersions20250619), + new(addSubQueryToBoards), + new(changeFixVersionsToText20260707), + new(addExtraJQLToScopeConfig), } } diff --git a/backend/plugins/jira/models/scope_config.go b/backend/plugins/jira/models/scope_config.go index a8bd78a981b..5dc6d07fed9 100644 --- a/backend/plugins/jira/models/scope_config.go +++ b/backend/plugins/jira/models/scope_config.go @@ -19,6 +19,7 @@ package models import ( "regexp" + "text/template" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/common" @@ -49,6 +50,7 @@ type JiraScopeConfig struct { TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` ApplicationType string `mapstructure:"applicationType,omitempty" json:"applicationType" gorm:"type:varchar(255)"` DueDateField string `mapstructure:"dueDateField,omitempty" json:"dueDateField" gorm:"type:varchar(255)"` + ExtraJQL string `mapstructure:"extraJql,omitempty" json:"extraJql" gorm:"type:varchar(255)"` } func (r *JiraScopeConfig) SetConnectionId(c *JiraScopeConfig, connectionId uint64) { @@ -73,6 +75,11 @@ func (r *JiraScopeConfig) Validate() errors.Error { return errors.Convert(err) } } + if r.ExtraJQL != "" { + if _, tmplErr := template.New("extraJql").Funcs(template.FuncMap{}).Option("missingkey=error").Parse(r.ExtraJQL); tmplErr != nil { + return errors.BadInput.Wrap(errors.Convert(tmplErr), "invalid ExtraJQL template") + } + } return nil } diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector.go b/backend/plugins/jira/tasks/board_filter_begin_collector.go index 6c513f1f83c..10dc60758a8 100644 --- a/backend/plugins/jira/tasks/board_filter_begin_collector.go +++ b/backend/plugins/jira/tasks/board_filter_begin_collector.go @@ -41,14 +41,18 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterBegin: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + + boardConfig, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfig.Filter.ID logger.Info("collect board filter:%s", filterId) - // get board filter jql + if boardConfig.SubQuery.Query != "" { + logger.Warn(nil, "board %d has kanban sub-filter: %s — using saved filter JQL for collection to avoid silent issue exclusion", data.Options.BoardId, boardConfig.SubQuery.Query) + } + filterInfo, err := getBoardFilterJql(data, filterId) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) @@ -62,17 +66,21 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { return errors.Default.Wrap(err, fmt.Sprintf("error finding record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + // Store filter ID and sub-query on task data for downstream subtasks + data.FilterId = filterId + record.SubQuery = boardConfig.SubQuery.Query + // full sync syncPolicy := taskCtx.TaskContext().SyncPolicy() if syncPolicy != nil && syncPolicy.FullSync { if record.Jql != jql { record.Jql = jql - err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) - if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) - } - logger.Info("full sync mode, update jql to %s", record.Jql) } + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } + logger.Info("full sync mode, update jql to %s", record.Jql) return nil } @@ -92,7 +100,6 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") if flag { logger.Info("connection_id:%d board_id:%d filter jql has changed, And the previous jql is %s, now jql is %s, run it in fullSync mode", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql) - // set full sync taskCtx.TaskContext().SetSyncPolicy(&coreModels.SyncPolicy{TriggerSyncPolicy: coreModels.TriggerSyncPolicy{FullSync: true}}) record.Jql = jql err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) @@ -102,24 +109,28 @@ func CollectBoardFilterBegin(taskCtx plugin.SubTaskContext) errors.Error { } else { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } + } else { + // JQL unchanged but sub-query may have changed — persist it + err = db.Update(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) + if err != nil { + return errors.Default.Wrap(err, fmt.Sprintf("error updating record in _tool_jira_boards table for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + } } - // no change return nil } -func getBoardFilterId(data *JiraTaskData) (string, error) { +func getBoardConfiguration(data *JiraTaskData) (*BoardConfiguration, error) { url := fmt.Sprintf("agile/1.0/board/%d/configuration", data.Options.BoardId) boardConfiguration, err := data.ApiClient.Get(url, nil, nil) if err != nil { - return "", err + return nil, err } bc := &BoardConfiguration{} err = helper.UnmarshalResponse(boardConfiguration, bc) if err != nil { - return "", err + return nil, err } - filterId := bc.Filter.ID - return filterId, nil + return bc, nil } func getBoardFilterJql(data *JiraTaskData, filterId string) (*FilterInfo, error) { @@ -141,6 +152,9 @@ type BoardConfiguration struct { Name string `json:"name"` Type string `json:"type"` Self string `json:"self"` + SubQuery struct { + Query string `json:"query"` + } `json:"subQuery"` Location struct { Type string `json:"type"` Key string `json:"key"` diff --git a/backend/plugins/jira/tasks/board_filter_begin_collector_test.go b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go new file mode 100644 index 00000000000..dc81004a971 --- /dev/null +++ b/backend/plugins/jira/tasks/board_filter_begin_collector_test.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" +) + +func Test_BoardConfiguration_UnmarshalSubQuery(t *testing.T) { + tests := []struct { + name string + raw string + wantSubQuery string + wantFilterID string + wantID int + wantName string + wantType string + wantColumnCount int + wantRankFieldID int + }{ + { + name: "kanban board with sub-filter object", + raw: `{"id":1201,"name":"Squad 5","type":"kanban",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/1201/configuration",` + + `"filter":{"id":"17696","self":"https://example.atlassian.net/rest/api/2/filter/17696"},` + + `"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},` + + `"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://example.atlassian.net/rest/api/2/status/1"}]},` + + `{"name":"Done","statuses":[{"id":"10037","self":"https://example.atlassian.net/rest/api/2/status/10037"}]}],` + + `"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "fixVersion in unreleasedVersions() OR fixVersion is EMPTY", + wantFilterID: "17696", + wantID: 1201, + wantName: "Squad 5", + wantType: "kanban", + wantColumnCount: 2, + wantRankFieldID: 10019, + }, + { + name: "board without subQuery field", + raw: `{"id":500,"name":"No SubFilter Board","type":"scrum",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/500/configuration",` + + `"filter":{"id":"99999","self":"https://example.atlassian.net/rest/api/2/filter/99999"},` + + `"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "", + wantFilterID: "99999", + wantID: 500, + wantName: "No SubFilter Board", + wantType: "scrum", + wantColumnCount: 0, + wantRankFieldID: 10019, + }, + { + name: "board with empty subQuery object", + raw: `{"id":600,"name":"Empty SubQuery Board","type":"kanban",` + + `"self":"https://example.atlassian.net/rest/agile/1.0/board/600/configuration",` + + `"filter":{"id":"11111","self":"https://example.atlassian.net/rest/api/2/filter/11111"},` + + `"subQuery":{},` + + `"columnConfig":{"columns":[],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}`, + wantSubQuery: "", + wantFilterID: "11111", + wantID: 600, + wantName: "Empty SubQuery Board", + wantType: "kanban", + wantColumnCount: 0, + wantRankFieldID: 10019, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bc BoardConfiguration + if err := json.Unmarshal([]byte(tt.raw), &bc); err != nil { + t.Fatalf("failed to unmarshal BoardConfiguration: %v", err) + } + if bc.SubQuery.Query != tt.wantSubQuery { + t.Errorf("SubQuery.Query = %q, want %q", bc.SubQuery.Query, tt.wantSubQuery) + } + if bc.Filter.ID != tt.wantFilterID { + t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, tt.wantFilterID) + } + if bc.ID != tt.wantID { + t.Errorf("ID = %d, want %d", bc.ID, tt.wantID) + } + if bc.Name != tt.wantName { + t.Errorf("Name = %q, want %q", bc.Name, tt.wantName) + } + if bc.Type != tt.wantType { + t.Errorf("Type = %q, want %q", bc.Type, tt.wantType) + } + if len(bc.ColumnConfig.Columns) != tt.wantColumnCount { + t.Errorf("ColumnConfig.Columns length = %d, want %d", len(bc.ColumnConfig.Columns), tt.wantColumnCount) + } + if bc.Ranking.RankCustomFieldID != tt.wantRankFieldID { + t.Errorf("Ranking.RankCustomFieldID = %d, want %d", bc.Ranking.RankCustomFieldID, tt.wantRankFieldID) + } + }) + } +} + +func Test_BoardConfiguration_FullJiraCloudResponse(t *testing.T) { + // Exact response payload from Jira Cloud for Board 1201 (Squad 5) + raw := `{"id":1201,"name":"Squad 5","type":"kanban","self":"https://rakutenadvertising.atlassian.net/rest/agile/1.0/board/1201/configuration","location":{"type":"user","id":"62d8159bb2e6b1992b5be875","self":"https://rakutenadvertising.atlassian.net/rest/api/2/user?accountId=62d8159bb2e6b1992b5be875"},"filter":{"id":"17696","self":"https://rakutenadvertising.atlassian.net/rest/api/2/filter/17696"},"subQuery":{"query":"fixVersion in unreleasedVersions() OR fixVersion is EMPTY"},"columnConfig":{"columns":[{"name":"Backlog","statuses":[{"id":"1","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/1"},{"id":"4","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/4"},{"id":"10016","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10016"},{"id":"10003","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10003"}]},{"name":"To Do","statuses":[{"id":"10054","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10054"}]},{"name":"Blocked","statuses":[{"id":"10019","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10019"}]},{"name":"In Development","statuses":[{"id":"10017","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10017"},{"id":"10177","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10177"},{"id":"10038","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10038"}]},{"name":"Code Review","statuses":[{"id":"10024","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10024"}]},{"name":"Ready for QA","statuses":[{"id":"10029","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10029"},{"id":"10033","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10033"}]},{"name":"In QA","statuses":[{"id":"10018","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10018"},{"id":"10158","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10158"}]},{"name":"Done","statuses":[{"id":"10037","self":"https://rakutenadvertising.atlassian.net/rest/api/2/status/10037"}]}],"constraintType":"issueCount"},"ranking":{"rankCustomFieldId":10019}}` + + var bc BoardConfiguration + if err := json.Unmarshal([]byte(raw), &bc); err != nil { + t.Fatalf("failed to unmarshal real Jira Cloud response: %v", err) + } + + if bc.SubQuery.Query != "fixVersion in unreleasedVersions() OR fixVersion is EMPTY" { + t.Errorf("SubQuery.Query = %q, want the fixVersion sub-filter", bc.SubQuery.Query) + } + if bc.Filter.ID != "17696" { + t.Errorf("Filter.ID = %q, want %q", bc.Filter.ID, "17696") + } + if len(bc.ColumnConfig.Columns) != 8 { + t.Errorf("ColumnConfig.Columns length = %d, want 8", len(bc.ColumnConfig.Columns)) + } + if bc.Location.ID != "62d8159bb2e6b1992b5be875" { + t.Errorf("Location.ID = %q, want %q", bc.Location.ID, "62d8159bb2e6b1992b5be875") + } +} diff --git a/backend/plugins/jira/tasks/board_filter_end_collector.go b/backend/plugins/jira/tasks/board_filter_end_collector.go index 65d8eca14fd..4dfe06fefea 100644 --- a/backend/plugins/jira/tasks/board_filter_end_collector.go +++ b/backend/plugins/jira/tasks/board_filter_end_collector.go @@ -40,14 +40,13 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { db := taskCtx.GetDal() logger.Info("collect board in collectBoardFilterEnd: %d", data.Options.BoardId) - // get board filter id - filterId, err := getBoardFilterId(data) + boardConfig, err := getBoardConfiguration(data) if err != nil { - return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter id for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) + return errors.Default.Wrap(err, fmt.Sprintf("error getting board configuration for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) } + filterId := boardConfig.Filter.ID logger.Info("collect board filter:%s", filterId) - // get board filter jql filterInfo, err := getBoardFilterJql(data, filterId) if err != nil { return errors.Default.Wrap(err, fmt.Sprintf("error getting board filter jql for connection_id:%d board_id:%d", data.Options.ConnectionId, data.Options.BoardId)) @@ -55,7 +54,6 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { jql := filterInfo.Jql logger.Info("collect board filter jql:%s", jql) - // should not change var record models.JiraBoard err = db.First(&record, dal.Where("connection_id = ? AND board_id = ? ", data.Options.ConnectionId, data.Options.BoardId)) if err != nil { @@ -63,12 +61,21 @@ func CollectBoardFilterEnd(taskCtx plugin.SubTaskContext) errors.Error { } logger.Info("get board filter jql:%s", record.Jql) + cfg := taskCtx.GetConfigReader() + autoRefresh := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") + if record.Jql != jql { - cfg := taskCtx.GetConfigReader() - flag := cfg.GetBool("JIRA_JQL_AUTO_FULL_REFRESH") - if !flag { + if !autoRefresh { return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d filter jql has changed, please use fullSync mode. And the previous jql is %s, now jql is %s", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql)) } + logger.Warn(nil, "connection_id:%d board_id:%d filter jql changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.Jql, jql) + } + + if record.SubQuery != boardConfig.SubQuery.Query { + logger.Warn(nil, "connection_id:%d board_id:%d board sub-filter changed during collection (previous: %s, now: %s)", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query) + if !autoRefresh { + return errors.Default.New(fmt.Sprintf("connection_id:%d board_id:%d board sub-filter has changed during collection, please use fullSync mode. Previous sub-filter: %s, now: %s", data.Options.ConnectionId, data.Options.BoardId, record.SubQuery, boardConfig.SubQuery.Query)) + } } return nil diff --git a/backend/plugins/jira/tasks/issue_collector.go b/backend/plugins/jira/tasks/issue_collector.go index 9a361cbbcca..fe1939b721b 100644 --- a/backend/plugins/jira/tasks/issue_collector.go +++ b/backend/plugins/jira/tasks/issue_collector.go @@ -18,20 +18,21 @@ limitations under the License. package tasks import ( + "bytes" "encoding/json" "fmt" "io" "net/http" "net/url" "strings" + "text/template" "time" "github.com/apache/incubator-devlake/core/dal" - "github.com/apache/incubator-devlake/plugins/jira/models" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" ) const RAW_ISSUE_TABLE = "jira_api_issues" @@ -51,78 +52,143 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ Ctx: taskCtx, - /* - This struct will be JSONEncoded and stored into database along with raw data itself, to identity minimal - set of data to be process, for example, we process JiraIssues by Board - */ Params: JiraApiParams{ ConnectionId: data.Options.ConnectionId, BoardId: data.Options.BoardId, }, - /* - Table store raw data - */ Table: RAW_ISSUE_TABLE, }) if err != nil { return err } - // build jql - // IMPORTANT: we have to keep paginated data in a consistence order to avoid data-missing, if we sort issues by - // `updated`, issue will be jumping between pages if it got updated during the collection process + // IMPORTANT: we sort by `created ASC` to keep paginated data in a consistent order. + // Sorting by `updated` would cause issues to jump between pages during collection. loc, err := getTimeZone(taskCtx) if err != nil { logger.Info("failed to get timezone, err: %v", err) } else { logger.Info("got user's timezone: %v", loc.String()) } - jql := "ORDER BY created ASC" + incrementalJql := "ORDER BY created ASC" if apiCollector.GetSince() != nil { - jql = buildJQL(*apiCollector.GetSince(), loc) - } - - err = apiCollector.InitCollector(api.ApiCollectorArgs{ - ApiClient: data.ApiClient, - PageSize: data.Options.PageSize, - /* - url may use arbitrary variables from different connection in any order, we need GoTemplate to allow more - flexible for all kinds of possibility. - Pager contains information for a particular page, calculated by ApiCollector, and will be passed into - GoTemplate to generate a url for that page. - We want to do page-fetching in ApiCollector, because the logic are highly similar, by doing so, we can - avoid duplicate logic for every tasks, and when we have a better idea like improving performance, we can - do it in one place - */ - UrlTemplate: "agile/1.0/board/{{ .Params.BoardId }}/issue", - /* - (Optional) Return query string for request, or you can plug them into UrlTemplate directly - */ + incrementalJql = buildJQL(*apiCollector.GetSince(), loc) + } + + // Use the search API with `filter = {id}` JQL instead of the board Agile API. + // The board Agile API applies kanban sub-filters server-side, which silently + // excludes resolved issues (e.g. those with a released fixVersion). + // The search API with the saved filter JQL returns all matching issues. + var extraJql string + if data.Options.ScopeConfig != nil && data.Options.ScopeConfig.ExtraJQL != "" { + renderedJql, renderErr := renderExtraJQL(data.Options.ScopeConfig.ExtraJQL, data) + if renderErr != nil { + return renderErr + } + extraJql = renderedJql + } + filterJql := buildFilterJQL(data.FilterId, extraJql, incrementalJql) + logger.Info("collecting issues via search API with JQL: %s", filterJql) + + pageSize := data.Options.PageSize + if pageSize == 0 { + pageSize = 100 + } + + if strings.EqualFold(string(data.JiraServerInfo.DeploymentType), string(models.DeploymentServer)) { + logger.Info("Using api/2/search for JIRA Server issue collection") + err = setupIssueV2Collector(apiCollector, data, filterJql, pageSize) + } else { + logger.Info("Using api/3/search/jql for JIRA Cloud issue collection") + err = setupIssueV3Collector(apiCollector, data, filterJql, pageSize) + } + if err != nil { + return err + } + + return apiCollector.Execute() +} + +// JqlTemplateData holds the variables available inside an ExtraJQL template. +// Users reference these with Go template syntax, e.g. `{{.BoardName}}`. +type JqlTemplateData struct { + BoardId uint64 // numeric ID of the connected Jira board + BoardName string // display name of the connected Jira board +} + +// renderExtraJQL executes the ExtraJQL scope-config field as a Go text/template, +// substituting board-level variables so the same scope config can produce +// different JQL for different boards. +// +// The template is parsed with an empty FuncMap (no built-in helpers such as +// printf) and missingkey=error so that typos in variable names produce an +// explicit error rather than silently rendering "". +func renderExtraJQL(tmplStr string, data *JiraTaskData) (string, errors.Error) { + tmpl, err := template.New("extraJql"). + Funcs(template.FuncMap{}). + Option("missingkey=error"). + Parse(tmplStr) + if err != nil { + return "", errors.BadInput.Wrap(err, "invalid ExtraJQL template") + } + + vars := JqlTemplateData{ + BoardId: data.Options.BoardId, + } + if data.Board != nil { + vars.BoardName = data.Board.Name + } + + var buf bytes.Buffer + if execErr := tmpl.Execute(&buf, vars); execErr != nil { + return "", errors.BadInput.Wrap(execErr, "failed to render ExtraJQL template") + } + return buf.String(), nil +} + +// buildFilterJQL composes a final JQL query from three inputs: +// - filterId: a Jira saved-filter ID (referenced via `filter = {id}`) +// - extraJql: optional user-supplied JQL fragment appended as an AND condition +// (e.g. `project = "MyComponent"`) to scope a large board down to one project +// - incrementalJql: the time-based clause generated by buildJQL, always ending +// with "ORDER BY created ASC" +// +// extraJql is wrapped in parentheses so that any OR/NOT operators inside it +// do not interfere with the surrounding AND chain. +func buildFilterJQL(filterId string, extraJql string, incrementalJql string) string { + const orderBy = "ORDER BY created ASC" + + var conditions []string + if filterId != "" { + conditions = append(conditions, fmt.Sprintf("filter = %s", filterId)) + } + if extraJql != "" { + conditions = append(conditions, fmt.Sprintf("(%s)", extraJql)) + } + if incrementalJql != orderBy { + // strip the trailing " ORDER BY created ASC" to isolate the time condition + conditions = append(conditions, strings.TrimSuffix(incrementalJql, " "+orderBy)) + } + + if len(conditions) == 0 { + return orderBy + } + return strings.Join(conditions, " AND ") + " " + orderBy +} + +func setupIssueV2Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: pageSize, + UrlTemplate: "api/2/search", Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} - query.Set("jql", jql) + query.Set("jql", filterJql) query.Set("startAt", fmt.Sprintf("%v", reqData.Pager.Skip)) query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) query.Set("expand", "changelog") return query, nil }, - /* - Some api might do pagination by http headers - */ - //Header: func(pager *plugin.Pager) http.Header { - //}, - /* - Sometimes, we need to collect data based on previous collected data, like jira changelog, it requires - issue_id as part of the url. - We can mimic `stdin` design, to accept a `Input` function which produces a `Iterator`, collector - should iterate all records, and do data-fetching for each on, either in parallel or sequential order - UrlTemplate: "api/3/issue/{{ Input.ID }}/changelog" - */ - //Input: databaseIssuesIterator, - /* - For api endpoint that returns number of total pages, ApiCollector can collect pages in parallel with ease, - or other techniques are required if this information was missing. - */ GetTotalPages: GetTotalPagesFromResponse, Concurrency: 10, ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { @@ -140,11 +206,40 @@ func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { return data.Issues, nil }, }) - if err != nil { - return err - } +} - return apiCollector.Execute() +func setupIssueV3Collector(apiCollector *api.StatefulApiCollector, data *JiraTaskData, filterJql string, pageSize int) errors.Error { + return apiCollector.InitCollector(api.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: pageSize, + UrlTemplate: "api/3/search/jql", + GetNextPageCustomData: getNextPageCustomDataForV3, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("jql", filterJql) + query.Set("maxResults", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("expand", "changelog") + query.Set("fields", "*all") + if reqData.CustomData != nil { + query.Set("nextPageToken", reqData.CustomData.(string)) + } + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var data struct { + Issues []json.RawMessage `json:"issues"` + } + blob, err := io.ReadAll(res.Body) + if err != nil { + return nil, errors.Convert(err) + } + err = json.Unmarshal(blob, &data) + if err != nil { + return nil, errors.Convert(err) + } + return data.Issues, nil + }, + }) } // buildJQL build jql based on timeAfter and incremental mode diff --git a/backend/plugins/jira/tasks/issue_collector_test.go b/backend/plugins/jira/tasks/issue_collector_test.go index 99bf5a53367..50c66947c59 100644 --- a/backend/plugins/jira/tasks/issue_collector_test.go +++ b/backend/plugins/jira/tasks/issue_collector_test.go @@ -20,6 +20,8 @@ package tasks import ( "testing" "time" + + "github.com/apache/incubator-devlake/plugins/jira/models" ) func Test_buildJQL(t *testing.T) { @@ -61,3 +63,148 @@ func Test_buildJQL(t *testing.T) { }) } } + +func Test_buildFilterJQL(t *testing.T) { + tests := []struct { + name string + filterId string + extraJql string + incrementalJql string + want string + }{ + { + name: "full sync with filter", + filterId: "12345", + incrementalJql: "ORDER BY created ASC", + want: "filter = 12345 ORDER BY created ASC", + }, + { + name: "incremental sync with filter", + filterId: "12345", + incrementalJql: "updated >= '2021/02/05 12:05' ORDER BY created ASC", + want: "filter = 12345 AND updated >= '2021/02/05 12:05' ORDER BY created ASC", + }, + { + name: "empty filter id falls back to incremental only", + filterId: "", + incrementalJql: "ORDER BY created ASC", + want: "ORDER BY created ASC", + }, + { + name: "empty filter id with incremental clause", + filterId: "", + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + }, + { + name: "extra jql with filter full sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql with filter incremental sync", + filterId: "12345", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `filter = 12345 AND (project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql without filter", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "ORDER BY created ASC", + want: `(project = "MyComponent") ORDER BY created ASC`, + }, + { + name: "extra jql without filter, incremental sync", + filterId: "", + extraJql: `project = "MyComponent"`, + incrementalJql: "updated >= '2024/01/01 00:00' ORDER BY created ASC", + want: `(project = "MyComponent") AND updated >= '2024/01/01 00:00' ORDER BY created ASC`, + }, + { + name: "extra jql with OR operator is parenthesized", + filterId: "12345", + extraJql: `project = "A" OR project = "B"`, + incrementalJql: "ORDER BY created ASC", + want: `filter = 12345 AND (project = "A" OR project = "B") ORDER BY created ASC`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := buildFilterJQL(tt.filterId, tt.extraJql, tt.incrementalJql); got != tt.want { + t.Errorf("buildFilterJQL() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_renderExtraJQL(t *testing.T) { + makeData := func(boardId uint64, boardName string, _ string) *JiraTaskData { + return &JiraTaskData{ + Options: &JiraOptions{BoardId: boardId}, + Board: &models.JiraBoard{BoardId: boardId, Name: boardName}, + } + } + + tests := []struct { + name string + tmpl string + data *JiraTaskData + want string + wantErr bool + }{ + { + name: "static JQL passes through unchanged", + tmpl: `project = "MyProject"`, + data: makeData(1, "My Board", ""), + want: `project = "MyProject"`, + }, + { + name: "BoardName substitution", + tmpl: `project = "{{.BoardName}}"`, + data: makeData(42, "Team Alpha", ""), + want: `project = "Team Alpha"`, + }, + { + name: "BoardId substitution", + tmpl: `cf[10001] = {{.BoardId}}`, + data: makeData(99, "Some Board", ""), + want: `cf[10001] = 99`, + }, + { + name: "nil Board falls back to empty BoardName", + tmpl: `project = "{{.BoardName}}"`, + data: &JiraTaskData{Options: &JiraOptions{BoardId: 1}, Board: nil}, + want: `project = ""`, + }, + { + name: "invalid template returns error", + tmpl: `project = "{{.Unclosed"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + { + name: "unknown field returns error (missingkey=error)", + tmpl: `component = "{{.Typo}}"`, + data: makeData(1, "My Board", ""), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := renderExtraJQL(tt.tmpl, tt.data) + if (err != nil) != tt.wantErr { + t.Errorf("renderExtraJQL() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && got != tt.want { + t.Errorf("renderExtraJQL() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/backend/plugins/jira/tasks/issue_extractor.go b/backend/plugins/jira/tasks/issue_extractor.go index d25423b918f..94620711399 100644 --- a/backend/plugins/jira/tasks/issue_extractor.go +++ b/backend/plugins/jira/tasks/issue_extractor.go @@ -163,7 +163,7 @@ func extractIssues(data *JiraTaskData, mappings *typeMappings, apiIssue *apiv2mo if value, ok := mappings.StandardStatusMappings[issue.Type][issue.StatusKey]; ok { issue.StdStatus = value.StandardStatus } - // issue commments + // issue comments results = append(results, issue) for _, comment := range comments { results = append(results, comment) diff --git a/backend/plugins/jira/tasks/task_data.go b/backend/plugins/jira/tasks/task_data.go index 1b0580396c5..3505bf162e8 100644 --- a/backend/plugins/jira/tasks/task_data.go +++ b/backend/plugins/jira/tasks/task_data.go @@ -37,6 +37,8 @@ type JiraTaskData struct { Options *JiraOptions ApiClient *api.ApiAsyncClient JiraServerInfo models.JiraServerInfo + FilterId string + Board *models.JiraBoard } type JiraApiParams models.JiraApiParams diff --git a/backend/plugins/linear/README.md b/backend/plugins/linear/README.md new file mode 100644 index 00000000000..e52b4527176 --- /dev/null +++ b/backend/plugins/linear/README.md @@ -0,0 +1,115 @@ + + +# Linear + +## Summary + +This plugin collects data from [Linear](https://linear.app) through its +[GraphQL API](https://linear.app/developers/graphql) and maps it into DevLake's +standardized `ticket` domain, so Linear issues appear in DevLake dashboards +(throughput, lead/cycle time, sprint burndown, etc.). + +The selectable **scope** is a Linear **Team**, which maps to a domain `Board`. + +## Supported data + +| Linear entity | Tool-layer table | Domain-layer table | +|-----------------|-----------------------------------|--------------------------------------------| +| Team | `_tool_linear_teams` (scope) | `boards` | +| User | `_tool_linear_accounts` | `accounts` | +| Workflow state | `_tool_linear_workflow_states` | (drives issue status mapping) | +| Issue | `_tool_linear_issues` | `issues`, `board_issues` | +| Label | `_tool_linear_issue_labels` | `issue_labels` | +| Comment | `_tool_linear_comments` | `issue_comments` | +| Cycle | `_tool_linear_cycles` | `sprints`, `board_sprints`, `sprint_issues`| +| Issue history | `_tool_linear_issue_history` | `issue_changelogs` | + +### Field mapping highlights + +- **Status** — derived deterministically from Linear's `WorkflowState.type` + (no manual mapping needed, unlike Jira): + - `backlog`, `unstarted` → `TODO` + - `started` → `IN_PROGRESS` + - `completed`, `canceled` → `DONE` +- **Priority** — Linear's integer priority maps to a label: `0` No priority, + `1` Urgent, `2` High, `3` Medium, `4` Low. +- **Type** — Linear has no native issue type, so issues default to `REQUIREMENT`. +- **Lead time** — `completedAt − createdAt` (Linear provides `startedAt`/`completedAt` + natively; the history changelog captures every status transition). +- **Story points** — Linear's `estimate`. + +## Authentication + +The plugin uses a Linear **personal API key**, passed verbatim in the +`Authorization` header (no `Bearer` prefix). Create one under +**Settings → Security & access → Personal API keys** in Linear. + +## Configuration + +Create a connection: + +``` +curl 'http://localhost:8080/api/plugins/linear/connections' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear", + "endpoint": "https://api.linear.app/graphql", + "token": "", + "rateLimitPerHour": 1500 +}' +``` + +Add a team scope (the team id is the Linear team UUID): + +``` +curl 'http://localhost:8080/api/plugins/linear/connections//scopes' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "data": [{ "connectionId": , "teamId": "", "name": "Engineering" }] +}' +``` + +## Collecting data + +``` +curl 'http://localhost:8080/api/pipelines' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear pipeline", + "plan": [[{ + "plugin": "linear", + "options": { "connectionId": , "teamId": "" } + }]] +}' +``` + +## Rate limiting + +Linear enforces a per-API-key request budget (1,500 requests/hour) plus a +complexity budget. The collector paces requests against the configured +`rateLimitPerHour` (default 1500). Issues are collected incrementally using +`updatedAt` ordering so re-runs only fetch changes. + +## Limitations / roadmap + +- Authentication is personal API key only; OAuth2 is a planned follow-up. +- Issue type defaults to `REQUIREMENT`; label-based type mapping via the scope + config is a planned follow-up. +- config-ui integration (connection form + team picker) and the website + documentation page are planned follow-ups; for now connections and scopes are + managed via the API calls shown above. diff --git a/backend/plugins/linear/api/blueprint_v200.go b/backend/plugins/linear/api/blueprint_v200.go new file mode 100644 index 00000000000..a26a43ab521 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "linear", + subtaskMetas, + scopeConfig.Entities, + tasks.LinearOptions{ + ConnectionId: connection.ID, + TeamId: scope.TeamId, + ScopeConfigId: scope.ScopeConfigId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.TeamId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/linear/api/blueprint_v200_test.go b/backend/plugins/linear/api/blueprint_v200_test.go new file mode 100644 index 00000000000..fe0d8b05db3 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200_test.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/stretchr/testify/assert" +) + +func mockLinearPlugin(t *testing.T) { + mockMeta := mockplugin.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/linear") + mockMeta.On("Name").Return("linear").Maybe() + _ = plugin.RegisterPlugin("linear", mockMeta) +} + +func TestMakeScopesV200(t *testing.T) { + mockLinearPlugin(t) + + const connectionId uint64 = 1 + const teamId = "team-1" + const expectDomainScopeId = "linear:LinearTeam:1:team-1" + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: connectionId}, + TeamId: teamId, + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: connectionId}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) +} + +func TestMakePipelinePlanV200PassesScopeConfigId(t *testing.T) { + const scopeConfigId uint64 = 42 + subtaskMetas := []plugin.SubTaskMeta{ + {Name: "convertIssues", EnabledByDefault: true, DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}}, + } + + plan, err := makePipelinePlanV200( + subtaskMetas, + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1, ScopeConfigId: scopeConfigId}, + TeamId: "team-1", + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(plan)) + assert.Equal(t, 1, len(plan[0])) + // the scope's scopeConfigId must be threaded into the task options so the + // convertor can resolve label-based issue-type mapping at runtime. + assert.EqualValues(t, scopeConfigId, plan[0][0].Options["scopeConfigId"]) +} + +func TestMakeScopesV200WithoutTicketEntity(t *testing.T) { + mockLinearPlugin(t) + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1}, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + // no ticket entity selected => no domain board scope produced + assert.Equal(t, 0, len(scopes)) +} diff --git a/backend/plugins/linear/api/connection_api.go b/backend/plugins/linear/api/connection_api.go new file mode 100644 index 00000000000..c2c95c54d64 --- /dev/null +++ b/backend/plugins/linear/api/connection_api.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +const defaultEndpoint = "https://api.linear.app/graphql" + +type LinearTestConnResponse struct { + shared.ApiBody + Connection *models.LinearConn +} + +func testConnection(ctx context.Context, connection models.LinearConn) (*LinearTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = defaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // Linear is GraphQL-over-HTTP-POST; a minimal viewer query verifies the key. + reqBody := map[string]interface{}{"query": "{ viewer { id name } }"} + res, err := apiClient.Post("", nil, reqBody, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API key") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := LinearTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test linear connection +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param body body models.LinearConn true "json body" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.LinearConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test linear connection by ID +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.LinearConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create linear connection +// @Summary create linear connection +// @Description Create linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch linear connection +// @Summary patch linear connection +// @Description Patch linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a linear connection +// @Summary delete a linear connection +// @Description Delete a linear connection +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all linear connections +// @Summary get all linear connections +// @Description Get all linear connections +// @Tags plugins/linear +// @Success 200 {object} []models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get linear connection detail +// @Summary get linear connection detail +// @Description Get linear connection detail +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go new file mode 100644 index 00000000000..850acf54825 --- /dev/null +++ b/backend/plugins/linear/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.LinearConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.LinearConnection, models.LinearTeam, models.LinearScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.LinearConnection) models.LinearConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.LinearConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination](raProxy, listLinearRemoteScopes) +} diff --git a/backend/plugins/linear/api/remote_api.go b/backend/plugins/linear/api/remote_api.go new file mode 100644 index 00000000000..094c785016d --- /dev/null +++ b/backend/plugins/linear/api/remote_api.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearRemotePagination drives cursor-based pagination through the GraphQL +// `teams` connection when listing remote scopes for the config UI. +type LinearRemotePagination struct { + Cursor string `json:"cursor"` +} + +// linearTeamsGraphqlResponse mirrors the shape of the `teams` query response. +type linearTeamsGraphqlResponse struct { + Data struct { + Teams struct { + Nodes []struct { + Id string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + Description string `json:"description"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + } `json:"teams"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const remoteScopesPageSize = 100 + +// listLinearRemoteScopes lists Linear teams as selectable scopes. Linear teams +// are a flat list, so there are no intermediate groups. +func listLinearRemoteScopes( + _ *models.LinearConnection, + apiClient plugin.ApiClient, + _ string, + page LinearRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], + nextPage *LinearRemotePagination, + err errors.Error, +) { + after := "" + if page.Cursor != "" { + after = fmt.Sprintf(", after: %q", page.Cursor) + } + query := fmt.Sprintf( + "query { teams(first: %d%s) { nodes { id name key description } pageInfo { hasNextPage endCursor } } }", + remoteScopesPageSize, after, + ) + + res, err := apiClient.Post("", nil, map[string]interface{}{"query": query}, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to query Linear teams") + } + var response linearTeamsGraphqlResponse + if err := api.UnmarshalResponse(res, &response); err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Linear teams response") + } + if len(response.Errors) > 0 { + return nil, nil, errors.Default.New("linear graphql teams query failed: " + response.Errors[0].Message) + } + + return mapLinearTeamsToScopeEntries(response), nextPageFrom(response), nil +} + +// mapLinearTeamsToScopeEntries converts a teams response into scope-list +// entries. Each team is a selectable (leaf) scope. +func mapLinearTeamsToScopeEntries(response linearTeamsGraphqlResponse) []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam] { + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], 0, len(response.Data.Teams.Nodes)) + for _, team := range response.Data.Teams.Nodes { + team := team + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: team.Id, + Name: team.Name, + FullName: team.Name, + Data: &models.LinearTeam{ + TeamId: team.Id, + Name: team.Name, + Key: team.Key, + Description: team.Description, + }, + }) + } + return children +} + +// nextPageFrom returns the cursor for the following page, or nil when the +// teams connection has been fully traversed. +func nextPageFrom(response linearTeamsGraphqlResponse) *LinearRemotePagination { + pageInfo := response.Data.Teams.PageInfo + if pageInfo.HasNextPage && pageInfo.EndCursor != "" { + return &LinearRemotePagination{Cursor: pageInfo.EndCursor} + } + return nil +} + +// RemoteScopes lists the Linear teams available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the Linear API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/linear/api/remote_api_test.go b/backend/plugins/linear/api/remote_api_test.go new file mode 100644 index 00000000000..801e0d5029f --- /dev/null +++ b/backend/plugins/linear/api/remote_api_test.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestMapLinearTeamsToScopeEntries(t *testing.T) { + var response linearTeamsGraphqlResponse + body := `{"data":{"teams":{"nodes":[` + + `{"id":"team-uuid-1","name":"Engineering","key":"ENG","description":"core eng"},` + + `{"id":"team-uuid-2","name":"Design","key":"DSG","description":""}` + + `],"pageInfo":{"hasNextPage":false,"endCursor":""}}}}` + assert.NoError(t, json.Unmarshal([]byte(body), &response)) + + entries := mapLinearTeamsToScopeEntries(response) + assert.Len(t, entries, 2) + + assert.Equal(t, api.RAS_ENTRY_TYPE_SCOPE, entries[0].Type) + assert.Nil(t, entries[0].ParentId) + assert.Equal(t, "team-uuid-1", entries[0].Id) + assert.Equal(t, "Engineering", entries[0].Name) + assert.Equal(t, "Engineering", entries[0].FullName) + // the scope payload must carry the team id used as the scope's primary key + assert.NotNil(t, entries[0].Data) + assert.Equal(t, "team-uuid-1", entries[0].Data.TeamId) + assert.Equal(t, "ENG", entries[0].Data.Key) + + assert.Equal(t, "team-uuid-2", entries[1].Id) + assert.Equal(t, "Design", entries[1].Name) +} + +func TestNextPageFrom(t *testing.T) { + var more linearTeamsGraphqlResponse + more.Data.Teams.PageInfo.HasNextPage = true + more.Data.Teams.PageInfo.EndCursor = "cursor-abc" + next := nextPageFrom(more) + assert.NotNil(t, next) + assert.Equal(t, "cursor-abc", next.Cursor) + + // no further pages -> nil, so the helper stops paginating + var last linearTeamsGraphqlResponse + last.Data.Teams.PageInfo.HasNextPage = false + last.Data.Teams.PageInfo.EndCursor = "cursor-xyz" + assert.Nil(t, nextPageFrom(last)) +} diff --git a/backend/plugins/linear/api/scope_api.go b/backend/plugins/linear/api/scope_api.go new file mode 100644 index 00000000000..9871d1d2bfe --- /dev/null +++ b/backend/plugins/linear/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.LinearTeam] +type ScopeDetail api.ScopeDetail[models.LinearTeam, models.LinearScopeConfig] + +// PutScopes create or update linear teams +// @Summary create or update linear teams +// @Description Create or update linear teams +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to linear team +// @Summary patch to linear team +// @Description patch to linear team +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Param scope body models.LinearTeam true "json" +// @Success 200 {object} models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get linear teams +// @Summary get linear teams +// @Description get linear teams +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one linear team +// @Summary get one linear team +// @Description get one linear team +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/linear/api/scope_config_api.go b/backend/plugins/linear/api/scope_config_api.go new file mode 100644 index 00000000000..e70b800fee2 --- /dev/null +++ b/backend/plugins/linear/api/scope_config_api.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for Linear +// @Summary create scope config for Linear +// @Description create scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for Linear +// @Summary update scope config for Linear +// @Description update scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/linear +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig return projects details related by scope config +// @Summary return all related projects +// @Description return all related projects +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ProjectScopeOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/linear/e2e/account_test.go b/backend/plugins/linear/e2e/account_test.go new file mode 100644 index 00000000000..9cb8f3a8e15 --- /dev/null +++ b/backend/plugins/linear/e2e/account_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearAccountDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_accounts.csv", "_raw_linear_accounts") + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearAccount{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(crossdomain.Account{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/board_test.go b/backend/plugins/linear/e2e/board_test.go new file mode 100644 index 00000000000..eb4e0487323 --- /dev/null +++ b/backend/plugins/linear/e2e/board_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearBoardDataFlow verifies that a Linear team scope is converted into a +// domain ticket.Board, keyed identically to the board_id that board_issues +// already reference (boardIdGen over LinearTeam). Without this, the boards table +// is empty and board-scoped dashboards return no data. +func TestLinearBoardDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the team scope lives in _tool_linear_teams (populated via the scope API) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_teams.csv", &models.LinearTeam{}) + + // convert: team scope -> domain board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertTeamsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/comment_test.go b/backend/plugins/linear/e2e/comment_test.go new file mode 100644 index 00000000000..67ee95adad5 --- /dev/null +++ b/backend/plugins/linear/e2e/comment_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCommentDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the comment convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_comments.csv", "_raw_linear_comments") + dataflowTester.FlushTabler(&models.LinearComment{}) + dataflowTester.Subtask(tasks.ExtractCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueComment{}) + dataflowTester.Subtask(tasks.ConvertCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/cycle_test.go b/backend/plugins/linear/e2e/cycle_test.go new file mode 100644 index 00000000000..207e9c8a3f9 --- /dev/null +++ b/backend/plugins/linear/e2e/cycle_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCycleDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // issues drive sprint_issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_cycles.csv", "_raw_linear_cycles") + dataflowTester.FlushTabler(&models.LinearCycle{}) + dataflowTester.Subtask(tasks.ExtractCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearCycle{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_cycles.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: cycles -> sprints + board_sprints + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Sprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardSprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: issues -> sprint_issues + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_leadtime_test.go b/backend/plugins/linear/e2e/issue_history_leadtime_test.go new file mode 100644 index 00000000000..7d4a4c9798a --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_leadtime_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueHistoryLeadTime proves lead time is derived from the recorded +// state transitions. issue-1 was started on 2026-05-02 and completed on +// 2026-05-03 (1440 min of active cycle time), even though its createdAt -> +// resolutionDate span is 2880 min. ConvertIssues sets the coarse 2880 fallback; +// ConvertIssueHistory must then refine it to 1440 from the history. Issues +// without the required transitions keep the fallback (issue-4 = 1440). +func TestLinearIssueHistoryLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // flush accounts so assignee/creator names stay empty in this lead-time- + // focused test regardless of other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + + // seed issues + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // seed history + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + + // ConvertIssues sets the createdAt -> resolutionDate fallback ... + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + + // ... then ConvertIssueHistory refines lead time from the transitions. + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_history_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_test.go b/backend/plugins/linear/e2e/issue_history_test.go new file mode 100644 index 00000000000..31f559694e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueHistoryDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the history convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssueHistory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_history.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_changelogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_incident_test.go b/backend/plugins/linear/e2e/issue_incident_test.go new file mode 100644 index 00000000000..fb214fa82e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_incident_test.go @@ -0,0 +1,67 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueIncidentMapping verifies label-based issue-type mapping. With a +// scope config whose IssueTypeIncident pattern matches the "Bug" label, ENG-1 +// (labeled "Bug") is converted to an INCIDENT, while issues without a matching +// label stay REQUIREMENT. This is what feeds DORA's change-failure-rate metric. +func TestLinearIssueIncidentMapping(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + IssueTypeIncident: "^Bug$", + }, + } + + // extract issues + inline labels from raw + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // accounts needed so the convertor can resolve assignee/creator names + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // convert -> domain issues; ENG-1 (Bug label) must be INCIDENT, others REQUIREMENT + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_incident.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go new file mode 100644 index 00000000000..783820f696d --- /dev/null +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueNegativeLeadTime guards the lead-time fallback against +// resolution timestamps that precede the creation timestamp (clock skew or +// migrated/imported issues). A negative duration must NOT be cast to uint -- +// doing so wraps to a huge bogus value. The expected behaviour is that no lead +// time is derived (lead_time_minutes stays empty). +func TestLinearIssueNegativeLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // extraction: raw -> tool layer + // Flush accounts so this lead-time-focused test is independent of any + // account rows left behind by other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_negative_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_longtitle_test.go b/backend/plugins/linear/e2e/issue_longtitle_test.go new file mode 100644 index 00000000000..f85062cdb8b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_longtitle_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" + "github.com/stretchr/testify/assert" +) + +// TestLinearIssueLongTitle guards against truncation/insert failure for long +// issue titles and URLs. Linear titles can exceed 255 chars (and the issue URL +// embeds a title slug), which overflowed the old varchar(255) columns and +// failed extraction with "Data too long for column 'title'". The columns are +// now untyped (longtext), matching the domain issues.title and jira's tool +// summary. +func TestLinearIssueLongTitle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ConnectionId: 1, TeamId: "team-1"}, + } + + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_long_title.csv", "_raw_linear_issues") + // must not error with "Data too long for column 'title'" + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + var issue models.LinearIssue + err := dataflowTester.Dal.First(&issue, dal.Where("connection_id = ? AND id = ?", 1, "issue-longtitle")) + assert.NoError(t, err) + assert.Len(t, issue.Title, 300, "full 300-char title must be stored untruncated") +} diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go new file mode 100644 index 00000000000..5f98aad8a61 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer (issues + inline labels) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(models.LinearIssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // accounts must be present so the convertor can resolve assignee/creator + // display names and emit issue_assignees rows. + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // verify conversion: tool layer -> domain layer (issues + board_issues + issue_assignees) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/label_test.go b/backend/plugins/linear/e2e/label_test.go new file mode 100644 index 00000000000..4861f6197da --- /dev/null +++ b/backend/plugins/linear/e2e/label_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearLabelDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // labels are produced inline by the issue extractor + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueLabel{}) + dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv new file mode 100644 index 00000000000..785c47d21b2 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-1"",""Name"":""alice"",""DisplayName"":""Alice Anderson"",""Email"":""alice@example.com"",""AvatarUrl"":""https://linear.app/avatars/alice.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-2"",""Name"":""bob"",""DisplayName"":""Bob Brown"",""Email"":""bob@example.com"",""AvatarUrl"":""https://linear.app/avatars/bob.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-3"",""Name"":""carol"",""DisplayName"":""Carol Clark"",""Email"":""carol@example.com"",""AvatarUrl"":""https://linear.app/avatars/carol.png"",""Active"":false}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv new file mode 100644 index 00000000000..6d6fc21e6a8 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv new file mode 100644 index 00000000000..8eb59ddb145 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-1"",""Number"":1,""Name"":"""",""StartsAt"":""2026-04-20T00:00:00Z"",""EndsAt"":""2026-05-04T00:00:00Z"",""CompletedAt"":""2026-05-04T00:00:00Z""}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-2"",""Number"":2,""Name"":""Sprint 2"",""StartsAt"":""2026-05-04T00:00:00Z"",""EndsAt"":""2026-05-18T00:00:00Z"",""CompletedAt"":null}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv new file mode 100644 index 00000000000..d1cef3c37ab --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv new file mode 100644 index 00000000000..f49f41c73b6 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-1"",""Identifier"":""ENG-1"",""Number"":1,""Title"":""Fix login bug"",""Description"":""Users cannot log in"",""Url"":""https://linear.app/eng/issue/ENG-1"",""Priority"":1,""Estimate"":3,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-03T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":""2026-05-03T00:00:00Z"",""CanceledAt"":null,""State"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""},""Assignee"":{""Id"":""user-1""},""Creator"":{""Id"":""user-2""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l1"",""Name"":""Bug""},{""Id"":""l2"",""Name"":""P1""}]}}",https://api.linear.app/graphql,null,2026-05-03 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-2"",""Identifier"":""ENG-2"",""Number"":2,""Title"":""Add dark mode"",""Description"":""Theme support"",""Url"":""https://linear.app/eng/issue/ENG-2"",""Priority"":2,""Estimate"":5,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l3"",""Name"":""Feature""}]}}",https://api.linear.app/graphql,null,2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-3"",""Identifier"":""ENG-3"",""Number"":3,""Title"":""Investigate flakiness"",""Description"":"""",""Url"":""https://linear.app/eng/issue/ENG-3"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-4"",""Identifier"":""ENG-4"",""Number"":4,""Title"":""Deprecated feature"",""Description"":""No longer needed"",""Url"":""https://linear.app/eng/issue/ENG-4"",""Priority"":3,""Estimate"":2,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T12:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-02T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":{""Id"":""user-3""},""Creator"":{""Id"":""user-1""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-02 12:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-5"",""Identifier"":""ENG-5"",""Number"":5,""Title"":""Write docs"",""Description"":""User guide"",""Url"":""https://linear.app/eng/issue/ENG-5"",""Priority"":4,""Estimate"":1,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T06:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""Assignee"":null,""Creator"":{""Id"":""user-2""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 06:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv new file mode 100644 index 00000000000..4842c33a61c --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-longtitle"",""Identifier"":""ENG-LONG"",""Number"":900,""Title"":""TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"",""Description"":""long title row"",""Url"":""https://linear.app/eng/issue/ENG-LONG/tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":null,""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv new file mode 100644 index 00000000000..f63750db475 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-neg"",""Identifier"":""ENG-NEG"",""Number"":99,""Title"":""Imported issue with skewed timestamps"",""Description"":""canceledAt precedes createdAt"",""Url"":""https://linear.app/eng/issue/ENG-NEG"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-10T00:00:00Z"",""UpdatedAt"":""2026-05-10T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-09T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-10 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv new file mode 100644 index 00000000000..72c0ddc877b --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog"",""Color"":""#bec2c8"",""Position"":0}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted"",""Color"":""#e2e2e2"",""Position"":1}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started"",""Color"":""#f2c94c"",""Position"":2}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed"",""Color"":""#5e6ad2"",""Position"":3}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled"",""Color"":""#95a2b3"",""Position"":4}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv new file mode 100644 index 00000000000..db07f6b4cf7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv @@ -0,0 +1,4 @@ +connection_id,id,name,display_name,email,avatar_url,active +1,user-1,alice,Alice Anderson,alice@example.com,https://linear.app/avatars/alice.png,1 +1,user-2,bob,Bob Brown,bob@example.com,https://linear.app/avatars/bob.png,1 +1,user-3,carol,Carol Clark,carol@example.com,https://linear.app/avatars/carol.png,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv new file mode 100644 index 00000000000..aec4878ecb1 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,body,author_id +1,comment-1,issue-1,Looking into this,user-2 +1,comment-2,issue-1,Fixed in PR 42,user-1 +1,comment-3,issue-2,Any update?,user-1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv new file mode 100644 index 00000000000..9e95c44d986 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv @@ -0,0 +1,3 @@ +connection_id,id,team_id,number,name,starts_at,ends_at,completed_at +1,cycle-1,team-1,1,,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00 +1,cycle-2,team-1,2,Sprint 2,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv new file mode 100644 index 00000000000..bc8fe9ec7ba --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,actor_id,from_state_id,from_state_name,from_state_type,to_state_id,to_state_name,to_state_type +1,hist-1,issue-1,user-2,state-backlog,Backlog,backlog,state-todo,Todo,unstarted +1,hist-2,issue-1,user-1,state-todo,Todo,unstarted,state-inprogress,In Progress,started +1,hist-3,issue-1,user-1,state-inprogress,In Progress,started,state-done,Done,completed diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv new file mode 100644 index 00000000000..20da8a076b0 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv @@ -0,0 +1,4 @@ +connection_id,issue_id,label_name +1,issue-1,Bug +1,issue-1,P1 +1,issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv new file mode 100644 index 00000000000..7c35d53d39e --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv new file mode 100644 index 00000000000..1541f9c95e4 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,team_id,name,key,description +1,0,team-1,Engineering,ENG,Core engineering team diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv new file mode 100644 index 00000000000..bd1d3443ae9 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,name,type,color,position +1,state-backlog,team-1,Backlog,backlog,#bec2c8,0 +1,state-canceled,team-1,Canceled,canceled,#95a2b3,4 +1,state-done,team-1,Done,completed,#5e6ad2,3 +1,state-inprogress,team-1,In Progress,started,#f2c94c,2 +1,state-todo,team-1,Todo,unstarted,#e2e2e2,1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv new file mode 100644 index 00000000000..12c2066d191 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,4 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status +linear:LinearAccount:1:user-1,alice@example.com,Alice Anderson,alice,https://linear.app/avatars/alice.png,,,1 +linear:LinearAccount:1:user-2,bob@example.com,Bob Brown,bob,https://linear.app/avatars/bob.png,,,1 +linear:LinearAccount:1:user-3,carol@example.com,Carol Clark,carol,https://linear.app/avatars/carol.png,,,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9a7d32f9477 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-1 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-2 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-3 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-4 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-5 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..ff0cfe33dc5 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,3 @@ +board_id,sprint_id +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-1 +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-2 diff --git a/backend/plugins/linear/e2e/snapshot_tables/boards.csv b/backend/plugins/linear/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4548d25c385 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +linear:LinearTeam:1:team-1,Engineering,Core engineering team,,,linear diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..38f569f7094 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,3 @@ +issue_id,assignee_id,assignee_name +linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,Alice Anderson +linear:LinearIssue:1:issue-4,linear:LinearAccount:1:user-3,Carol Clark diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..9d53280010f --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,4 @@ +id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date +linear:LinearIssueHistory:1:hist-1,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-2,,state,status,Backlog,Todo,TODO,TODO,2026-05-01T08:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-2,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,Todo,In Progress,TODO,IN_PROGRESS,2026-05-02T00:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-3,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,In Progress,Done,IN_PROGRESS,DONE,2026-05-03T00:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv new file mode 100644 index 00000000000..925e84e5648 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv @@ -0,0 +1,4 @@ +id,issue_id,body,account_id,created_date,updated_date +linear:LinearComment:1:comment-1,linear:LinearIssue:1:issue-1,Looking into this,linear:LinearAccount:1:user-2,2026-05-02T10:00:00.000+00:00,2026-05-02T10:00:00.000+00:00 +linear:LinearComment:1:comment-2,linear:LinearIssue:1:issue-1,Fixed in PR 42,linear:LinearAccount:1:user-1,2026-05-03T09:00:00.000+00:00,2026-05-03T09:30:00.000+00:00 +linear:LinearComment:1:comment-3,linear:LinearIssue:1:issue-2,Any update?,linear:LinearAccount:1:user-1,2026-05-02T11:00:00.000+00:00,2026-05-02T11:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv new file mode 100644 index 00000000000..e7c911e0005 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv @@ -0,0 +1,4 @@ +issue_id,label_name +linear:LinearIssue:1:issue-1,Bug +linear:LinearIssue:1:issue-1,P1 +linear:LinearIssue:1:issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..f55c39e0eb8 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv new file mode 100644 index 00000000000..d11625e3c3d --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv new file mode 100644 index 00000000000..1adc9cab7da --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,INCIDENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv new file mode 100644 index 00000000000..4375d47901b --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-neg,https://linear.app/eng/issue/ENG-NEG,,ENG-NEG,Imported issue with skewed timestamps,canceledAt precedes createdAt,,REQUIREMENT,,DONE,Canceled,,2026-05-09T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..b13b4f85210 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,3 @@ +sprint_id,issue_id +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-1 +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-4 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv new file mode 100644 index 00000000000..1074725b86c --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv @@ -0,0 +1 @@ +sprint_id,issue_id diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..a09ab2ffe70 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,3 @@ +id,name,url,status,started_date,ended_date,completed_date,original_board_id +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 diff --git a/backend/plugins/linear/e2e/sprint_issue_test.go b/backend/plugins/linear/e2e/sprint_issue_test.go new file mode 100644 index 00000000000..81e4508e43f --- /dev/null +++ b/backend/plugins/linear/e2e/sprint_issue_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearSprintIssueStaleCycle guards against stale sprint_issues rows when +// an issue is moved out of a cycle. Sprint membership is derived from the +// issue's cycle_id; once that empties on re-collection, the issue must no +// longer appear in sprint_issues from a previous run. +func TestLinearSprintIssueStaleCycle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // seed issues: issue-1 and issue-4 both belong to cycle-1 + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // first conversion: both issues land in the sprint + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + + // every issue is moved out of its cycle; on re-collection cycle_id empties. + // The second conversion then produces zero sprint issues, which is the case + // the batch divider's lazy delete fails to cover. + if err := dataflowTester.Dal.Exec( + "UPDATE _tool_linear_issues SET cycle_id = '' WHERE connection_id = ? AND team_id = ?", 1, "team-1", + ); err != nil { + t.Fatal(err) + } + + // second conversion (no flush) must drop ALL stale sprint_issues rows + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues_after_leaving_cycle.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/workflow_state_test.go b/backend/plugins/linear/e2e/workflow_state_test.go new file mode 100644 index 00000000000..5af5c4a66b9 --- /dev/null +++ b/backend/plugins/linear/e2e/workflow_state_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearWorkflowStateDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_workflow_states.csv", "_raw_linear_workflow_states") + dataflowTester.FlushTabler(&models.LinearWorkflowState{}) + dataflowTester.Subtask(tasks.ExtractWorkflowStatesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearWorkflowState{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_workflow_states.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go new file mode 100644 index 00000000000..e1bc9849131 --- /dev/null +++ b/backend/plugins/linear/impl/impl.go @@ -0,0 +1,233 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Linear)(nil) + +type Linear struct{} + +func (p Linear) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Linear) Description() string { + return "To collect and enrich data from Linear" +} + +func (p Linear) Name() string { + return "linear" +} + +func (p Linear) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linear" +} + +func (p Linear) Connection() dal.Tabler { + return &models.LinearConnection{} +} + +func (p Linear) Scope() plugin.ToolLayerScope { + return &models.LinearTeam{} +} + +func (p Linear) ScopeConfig() dal.Tabler { + return &models.LinearScopeConfig{} +} + +func (p Linear) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linear) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.LinearConnection{}, + &models.LinearTeam{}, + &models.LinearScopeConfig{}, + &models.LinearAccount{}, + &models.LinearIssue{}, + &models.LinearComment{}, + &models.LinearIssueLabel{}, + &models.LinearWorkflowState{}, + &models.LinearCycle{}, + &models.LinearIssueHistory{}, + } +} + +func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectAccountsMeta, + tasks.ExtractAccountsMeta, + tasks.CollectWorkflowStatesMeta, + tasks.ExtractWorkflowStatesMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, + tasks.CollectCommentsMeta, + tasks.ExtractCommentsMeta, + tasks.CollectCyclesMeta, + tasks.ExtractCyclesMeta, + tasks.CollectIssueHistoryMeta, + tasks.ExtractIssueHistoryMeta, + tasks.ConvertTeamsMeta, + tasks.ConvertAccountsMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, + tasks.ConvertCommentsMeta, + tasks.ConvertCyclesMeta, + tasks.ConvertSprintIssuesMeta, + tasks.ConvertIssueHistoryMeta, + } +} + +func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.LinearOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode Linear options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("linear connectionId is invalid") + } + if op.TeamId == "" { + return nil, errors.BadInput.New("linear teamId is required") + } + + connection := &models.LinearConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Linear plugin") + } + + graphqlClient, err := tasks.NewLinearGraphqlClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create Linear GraphQL client") + } + + // Resolve the scope config (label-based issue-type mapping). Default to an + // empty config when none is set so subtasks can rely on it being non-nil. + scopeConfig := &models.LinearScopeConfig{} + if op.ScopeConfigId != 0 { + if err := taskCtx.GetDal().First(scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); err != nil { + return nil, errors.Default.Wrap(err, "error getting scope config for Linear plugin") + } + } + + taskData := &tasks.LinearTaskData{ + Options: &op, + GraphqlClient: graphqlClient, + ScopeConfig: scopeConfig, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Linear) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Linear) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.LinearTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.GraphqlClient != nil { + data.GraphqlClient.Release() + } + return nil +} diff --git a/backend/plugins/linear/impl/impl_test.go b/backend/plugins/linear/impl/impl_test.go new file mode 100644 index 00000000000..d11b3ec5ae1 --- /dev/null +++ b/backend/plugins/linear/impl/impl_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestApiResourcesRegistersRemoteScopes guards the remote-scopes endpoints used +// by the config UI to enumerate Linear teams. +func TestApiResourcesRegistersRemoteScopes(t *testing.T) { + resources := Linear{}.ApiResources() + + remoteScopes, ok := resources["connections/:connectionId/remote-scopes"] + assert.True(t, ok, "remote-scopes route must be registered") + assert.NotNil(t, remoteScopes["GET"], "remote-scopes must handle GET") + + proxy, ok := resources["connections/:connectionId/proxy/rest/*path"] + assert.True(t, ok, "proxy route must be registered") + assert.NotNil(t, proxy["GET"], "proxy must handle GET") +} diff --git a/backend/plugins/linear/linear.go b/backend/plugins/linear/linear.go new file mode 100644 index 00000000000..2cb64a49470 --- /dev/null +++ b/backend/plugins/linear/linear.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Linear //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "linear"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "linear connection id") + teamId := cmd.Flags().StringP("team", "t", "", "linear team id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("team") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "teamId": *teamId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linear/models/account.go b/backend/plugins/linear/models/account.go new file mode 100644 index 00000000000..7d1c9376f98 --- /dev/null +++ b/backend/plugins/linear/models/account.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearAccount is a Linear user (tool layer), converted to crossdomain.Account. +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Name string `gorm:"type:varchar(255)" json:"name"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Email string `gorm:"type:varchar(255)" json:"email"` + AvatarUrl string `gorm:"type:varchar(255)" json:"avatarUrl"` + Active bool `json:"active"` + common.NoPKModel +} + +func (LinearAccount) TableName() string { + return "_tool_linear_accounts" +} diff --git a/backend/plugins/linear/models/connection.go b/backend/plugins/linear/models/connection.go new file mode 100644 index 00000000000..6f63e431a11 --- /dev/null +++ b/backend/plugins/linear/models/connection.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// LinearConn holds the essential information to connect to the Linear API. +// Linear authenticates with a personal API key passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. +type LinearConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the Linear API. +func (lc *LinearConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", lc.Token) + return nil +} + +func (lc *LinearConn) Sanitize() LinearConn { + lc.Token = utils.SanitizeString(lc.Token) + return *lc +} + +// LinearConnection holds LinearConn plus ID/Name for database storage. +type LinearConnection struct { + helper.BaseConnection `mapstructure:",squash"` + LinearConn `mapstructure:",squash"` +} + +func (connection LinearConnection) Sanitize() LinearConnection { + connection.LinearConn = connection.LinearConn.Sanitize() + return connection +} + +func (connection *LinearConnection) MergeFromRequest(target *LinearConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (LinearConnection) TableName() string { + return "_tool_linear_connections" +} diff --git a/backend/plugins/linear/models/cycle.go b/backend/plugins/linear/models/cycle.go new file mode 100644 index 00000000000..4f61370dbb2 --- /dev/null +++ b/backend/plugins/linear/models/cycle.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearCycle is a Linear cycle (sprint-equivalent), converted to ticket.Sprint. +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Number int `json:"number"` + Name string `gorm:"type:varchar(255)" json:"name"` + StartsAt *time.Time `json:"startsAt"` + EndsAt *time.Time `json:"endsAt"` + CompletedAt *time.Time `json:"completedAt"` + common.NoPKModel +} + +func (LinearCycle) TableName() string { + return "_tool_linear_cycles" +} diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go new file mode 100644 index 00000000000..991585de956 --- /dev/null +++ b/backend/plugins/linear/models/issue.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssue is the tool-layer representation of a Linear issue. +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `json:"title"` + Description string `json:"description"` + Url string `json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` + common.NoPKModel +} + +func (LinearIssue) TableName() string { + return "_tool_linear_issues" +} diff --git a/backend/plugins/linear/models/issue_comment.go b/backend/plugins/linear/models/issue_comment.go new file mode 100644 index 00000000000..8ba18505038 --- /dev/null +++ b/backend/plugins/linear/models/issue_comment.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearComment is the tool-layer representation of a comment on a Linear issue. +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + Body string `json:"body"` + AuthorId string `gorm:"type:varchar(255)" json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + common.NoPKModel +} + +func (LinearComment) TableName() string { + return "_tool_linear_comments" +} diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go new file mode 100644 index 00000000000..027f6cb4a07 --- /dev/null +++ b/backend/plugins/linear/models/issue_history.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueHistory is a single entry in a Linear issue's history, used to +// build domain-layer changelogs and derive lead/cycle time. +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + ActorId string `gorm:"type:varchar(255)" json:"actorId"` + FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` + FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + FromStateType string `gorm:"type:varchar(100)" json:"fromStateType"` + ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` + ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + ToStateType string `gorm:"type:varchar(100)" json:"toStateType"` + CreatedAt time.Time `gorm:"index" json:"createdAt"` + common.NoPKModel +} + +func (LinearIssueHistory) TableName() string { + return "_tool_linear_issue_history" +} diff --git a/backend/plugins/linear/models/issue_label.go b/backend/plugins/linear/models/issue_label.go new file mode 100644 index 00000000000..76a4517cb7b --- /dev/null +++ b/backend/plugins/linear/models/issue_label.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueLabel joins a Linear issue to one of its labels. Labels are +// collected inline with issues, so there is no separate label collector. +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)" json:"issueId"` + LabelName string `gorm:"primaryKey;type:varchar(255)" json:"labelName"` + common.NoPKModel +} + +func (LinearIssueLabel) TableName() string { + return "_tool_linear_issue_labels" +} diff --git a/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go new file mode 100644 index 00000000000..9bb577343be --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.LinearConnection{}, + &archived.LinearTeam{}, + &archived.LinearScopeConfig{}, + &archived.LinearAccount{}, + &archived.LinearIssue{}, + &archived.LinearComment{}, + &archived.LinearIssueLabel{}, + &archived.LinearWorkflowState{}, + &archived.LinearCycle{}, + &archived.LinearIssueHistory{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260601000001 +} + +func (*addInitTables) Name() string { + return "linear init schemas" +} diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..a90c3217153 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -0,0 +1,169 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/linear/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type LinearConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (LinearConnection) TableName() string { return "_tool_linear_connections" } + +type LinearTeam struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + TeamId string `json:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + Key string `json:"key" gorm:"type:varchar(255)"` + Description string `json:"description"` +} + +func (LinearTeam) TableName() string { return "_tool_linear_teams" } + +type LinearScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + IssueTypeRequirement string `json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { return "_tool_linear_scope_configs" } + +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + DisplayName string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + Active bool + archived.NoPKModel +} + +func (LinearAccount) TableName() string { return "_tool_linear_accounts" } + +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string + Description string + Url string + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + archived.NoPKModel +} + +func (LinearIssue) TableName() string { return "_tool_linear_issues" } + +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + Body string + AuthorId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearComment) TableName() string { return "_tool_linear_comments" } + +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + LabelName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (LinearIssueLabel) TableName() string { return "_tool_linear_issue_labels" } + +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Type string `gorm:"type:varchar(100)"` + Color string `gorm:"type:varchar(50)"` + Position float64 + archived.NoPKModel +} + +func (LinearWorkflowState) TableName() string { return "_tool_linear_workflow_states" } + +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Number int + Name string `gorm:"type:varchar(255)"` + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time + archived.NoPKModel +} + +func (LinearCycle) TableName() string { return "_tool_linear_cycles" } + +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + ActorId string `gorm:"type:varchar(255)"` + FromStateId string `gorm:"type:varchar(255)"` + FromStateName string `gorm:"type:varchar(255)"` + FromStateType string `gorm:"type:varchar(100)"` + ToStateId string `gorm:"type:varchar(255)"` + ToStateName string `gorm:"type:varchar(255)"` + ToStateType string `gorm:"type:varchar(100)"` + CreatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearIssueHistory) TableName() string { return "_tool_linear_issue_history" } diff --git a/backend/plugins/linear/models/migrationscripts/register.go b/backend/plugins/linear/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/linear/models/scope_config.go b/backend/plugins/linear/models/scope_config.go new file mode 100644 index 00000000000..309ce749eae --- /dev/null +++ b/backend/plugins/linear/models/scope_config.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearScopeConfig keeps status mapping deterministic (Linear's +// WorkflowState.type maps to TODO/IN_PROGRESS/DONE without user input) but +// allows label-based issue-type mapping. Linear has no native issue "type", so +// each pattern below is a regular expression matched against an issue's label +// names to derive the domain ticket.Issue.Type. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue matching none defaults to REQUIREMENT. +type LinearScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { + return "_tool_linear_scope_configs" +} + +func (sc *LinearScopeConfig) SetConnectionId(c *LinearScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/linear/models/team.go b/backend/plugins/linear/models/team.go new file mode 100644 index 00000000000..d6226208266 --- /dev/null +++ b/backend/plugins/linear/models/team.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*LinearTeam)(nil) + +// LinearTeam is the data-source scope for the Linear plugin. A Linear Team +// owns issues, cycles, workflow states and labels, mapping cleanly to a +// DevLake domain-layer ticket.Board. +type LinearTeam struct { + common.Scope `mapstructure:",squash"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description"` +} + +func (t LinearTeam) ScopeId() string { + return t.TeamId +} + +func (t LinearTeam) ScopeName() string { + return t.Name +} + +func (t LinearTeam) ScopeFullName() string { + return t.Name +} + +func (t LinearTeam) ScopeParams() interface{} { + return &LinearApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (LinearTeam) TableName() string { + return "_tool_linear_teams" +} + +// LinearApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_linear_* table. +type LinearApiParams struct { + ConnectionId uint64 + TeamId string +} diff --git a/backend/plugins/linear/models/workflow_state.go b/backend/plugins/linear/models/workflow_state.go new file mode 100644 index 00000000000..9273f183c91 --- /dev/null +++ b/backend/plugins/linear/models/workflow_state.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearWorkflowState is a Linear team's workflow state. Its Type +// (backlog|unstarted|started|completed|canceled) drives issue status mapping. +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Type string `gorm:"type:varchar(100)" json:"type"` + Color string `gorm:"type:varchar(50)" json:"color"` + Position float64 `json:"position"` + common.NoPKModel +} + +func (LinearWorkflowState) TableName() string { + return "_tool_linear_workflow_states" +} diff --git a/backend/plugins/linear/tasks/account_collector.go b/backend/plugins/linear/tasks/account_collector.go new file mode 100644 index 00000000000..88fa181f002 --- /dev/null +++ b/backend/plugins/linear/tasks/account_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ACCOUNTS_TABLE = "linear_accounts" + +// GraphqlQueryAccountWrapper is the paginated `users` query envelope. +type GraphqlQueryAccountWrapper struct { + Users struct { + Nodes []GraphqlQueryAccount + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"users(first: $pageSize, after: $skipCursor)"` +} + +type GraphqlQueryAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string + Active bool +} + +var CollectAccountsMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectAccounts, + EnabledByDefault: true, + Description: "Collect workspace users from the Linear GraphQL API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectAccounts + +func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryAccountWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryAccountWrapper) + return query.Users.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryAccountWrapper) + for _, account := range query.Users.Nodes { + messages = append(messages, errors.Must1(json.Marshal(account))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/account_convertor.go b/backend/plugins/linear/tasks/account_convertor.go new file mode 100644 index 00000000000..d3c8efd3686 --- /dev/null +++ b/backend/plugins/linear/tasks/account_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_accounts into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.LinearAccount{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertAccounts + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearAccount{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearAccount{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + account := inputRow.(*models.LinearAccount) + status := 1 + if !account.Active { + status = 0 + } + fullName := account.Name + if account.DisplayName != "" { + fullName = account.DisplayName + } + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, account.Id), + }, + UserName: account.Name, + FullName: fullName, + Email: account.Email, + AvatarUrl: account.AvatarUrl, + Status: status, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/account_extractor.go b/backend/plugins/linear/tasks/account_extractor.go new file mode 100644 index 00000000000..19cc86a04cc --- /dev/null +++ b/backend/plugins/linear/tasks/account_extractor.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractAccountsMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractAccounts, + EnabledByDefault: true, + Description: "Extract raw user data into tool layer table _tool_linear_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractAccounts + +func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiAccount := &GraphqlQueryAccount{} + if err := errors.Convert(json.Unmarshal(row.Data, apiAccount)); err != nil { + return nil, err + } + if apiAccount.Id == "" { + return nil, nil + } + account := &models.LinearAccount{ + ConnectionId: data.Options.ConnectionId, + Id: apiAccount.Id, + Name: apiAccount.Name, + DisplayName: apiAccount.DisplayName, + Email: apiAccount.Email, + AvatarUrl: apiAccount.AvatarUrl, + Active: apiAccount.Active, + } + return []interface{}{account}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/api_client.go b/backend/plugins/linear/tasks/api_client.go new file mode 100644 index 00000000000..0ca6b8da25a --- /dev/null +++ b/backend/plugins/linear/tasks/api_client.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + gocontext "context" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +// linearTransport injects the Linear personal API key into every request. +// Linear expects the key verbatim in the Authorization header (no Bearer prefix). +type linearTransport struct { + token string + base http.RoundTripper +} + +func (t *linearTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", t.token) + return t.base.RoundTrip(req) +} + +// graphqlQueryViewer is a tiny probe used to validate connectivity / liveness. +type graphqlQueryViewer struct { + Viewer struct { + Id graphql.String + } +} + +// defaultRateLimitPerHour is Linear's documented per-API-key request budget. +// Used when the connection does not override RateLimitPerHour. +const defaultRateLimitPerHour = 1500 + +// NewLinearGraphqlClient builds a rate-limited async GraphQL client for the +// Linear API from the given connection. +func NewLinearGraphqlClient(taskCtx plugin.TaskContext, connection *models.LinearConnection) (*helper.GraphqlAsyncClient, errors.Error) { + httpClient, err := newLinearHttpClient(connection) + if err != nil { + return nil, err + } + + endpoint := connection.Endpoint + if endpoint == "" { + endpoint = "https://api.linear.app/graphql" + } + client := graphql.NewClient(endpoint, httpClient) + + rateLimitPerHour := connection.RateLimitPerHour + if rateLimitPerHour <= 0 { + rateLimitPerHour = defaultRateLimitPerHour + } + + return helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + func(ctx gocontext.Context, c *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, e errors.Error) { + // Linear does not expose rate-limit info in the GraphQL body (it uses + // HTTP response headers), so we probe liveness and pace against the + // configured hourly budget. The async client self-throttles from here. + var q graphqlQueryViewer + dataErrors, queryErr := errors.Convert01(c.Query(ctx, &q, nil)) + if queryErr != nil { + return 0, nil, queryErr + } + if len(dataErrors) > 0 { + return 0, nil, errors.Default.Wrap(dataErrors[0], "linear graphql viewer query failed") + } + reset := time.Now().Add(1 * time.Hour) + logger.Info("linear graphql client initialized, pacing against %d req/hour", rateLimitPerHour) + return rateLimitPerHour, &reset, nil + }) +} + +func newLinearHttpClient(connection *models.LinearConnection) (*http.Client, errors.Error) { + base := http.DefaultTransport + if proxy := connection.Proxy; proxy != "" { + pu, err := url.Parse(proxy) + if err != nil { + return nil, errors.BadInput.Wrap(err, "malformed proxy url") + } + base = &http.Transport{Proxy: http.ProxyURL(pu)} + } + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &linearTransport{token: connection.Token, base: base}, + }, nil +} diff --git a/backend/plugins/linear/tasks/board_convertor.go b/backend/plugins/linear/tasks/board_convertor.go new file mode 100644 index 00000000000..9366efed2ab --- /dev/null +++ b/backend/plugins/linear/tasks/board_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// RAW_TEAMS_TABLE labels the raw-data lineage for the team-scope-derived board. +// Teams are added as scopes (no collector), so this is a logical tag only. +const RAW_TEAMS_TABLE = "linear_teams" + +var ConvertTeamsMeta = plugin.SubTaskMeta{ + Name: "Convert Teams", + EntryPoint: ConvertTeams, + EnabledByDefault: true, + Description: "Convert the Linear team scope (_tool_linear_teams) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearTeam{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTeams + +func ConvertTeams(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the issue/sprint convertors so the + // board joins to the board_issues/sprint_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearTeam{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_TEAMS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearTeam{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + team := inputRow.(*models.LinearTeam) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, team.TeamId)}, + Name: team.Name, + Description: team.Description, + Type: "linear", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go new file mode 100644 index 00000000000..14e801ff9e3 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -0,0 +1,148 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_COMMENTS_TABLE = "linear_comments" + +// SimpleLinearIssue is the iterator element used to drive per-issue collection +// of child resources (comments, history). Its JSON form is stored in the raw +// row's `input` column so extractors can recover the owning issue id. +type SimpleLinearIssue struct { + // Id is populated by the DalCursorIterator (the _tool_linear_issues.id column) + // when driving per-issue child collection. + Id string `json:"Id"` + // IssueId is populated when parsing a raw row's `input` column: the GraphQL + // collector stores the query variables there (which carry `issueId`), not the + // iterator element. OwningIssueId resolves whichever is present. + IssueId string `json:"issueId" gorm:"-"` +} + +// OwningIssueId returns the issue id this child row belongs to, tolerating both +// the iterator element shape ({"Id":...}) and the collector's stored variables +// shape ({"issueId":...}). +func (s SimpleLinearIssue) OwningIssueId() string { + if s.IssueId != "" { + return s.IssueId + } + return s.Id +} + +// GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. +type GraphqlQueryCommentWrapper struct { + Issue struct { + Comments struct { + Nodes []GraphqlQueryComment + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"comments(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryComment struct { + Id string + Body string + CreatedAt time.Time + UpdatedAt time.Time + User *struct{ Id string } +} + +var CollectCommentsMeta = plugin.SubTaskMeta{ + Name: "Collect Comments", + EntryPoint: CollectComments, + EnabledByDefault: true, + Description: "Collect comments for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectComments + +func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's comments cannot have changed, so re-fetching every + // issue each run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCommentWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCommentWrapper) + return query.Issue.Comments.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCommentWrapper) + for _, comment := range query.Issue.Comments.Nodes { + messages = append(messages, errors.Must1(json.Marshal(comment))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_convertor.go b/backend/plugins/linear/tasks/comment_convertor.go new file mode 100644 index 00000000000..8c4729f0e5c --- /dev/null +++ b/backend/plugins/linear/tasks/comment_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCommentsMeta = plugin.SubTaskMeta{ + Name: "Convert Comments", + EntryPoint: ConvertComments, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_comments into domain layer table issue_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearComment{}.TableName(), RAW_COMMENTS_TABLE}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertComments + +func ConvertComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + commentIdGen := didgen.NewDomainIdGenerator(&models.LinearComment{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("c.*"), + dal.From("_tool_linear_comments c"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = c.connection_id AND i.id = c.issue_id)"), + dal.Where("c.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearComment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + comment := inputRow.(*models.LinearComment) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(connectionId, comment.Id)}, + IssueId: issueIdGen.Generate(connectionId, comment.IssueId), + Body: comment.Body, + CreatedDate: comment.CreatedAt, + } + if comment.AuthorId != "" { + domainComment.AccountId = accountIdGen.Generate(connectionId, comment.AuthorId) + } + if !comment.UpdatedAt.IsZero() { + domainComment.UpdatedDate = &comment.UpdatedAt + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go new file mode 100644 index 00000000000..15b28a3b969 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCommentsMeta = plugin.SubTaskMeta{ + Name: "Extract Comments", + EntryPoint: ExtractComments, + EnabledByDefault: true, + Description: "Extract raw comment data into tool layer table _tool_linear_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractComments + +func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiComment := &GraphqlQueryComment{} + if err := errors.Convert(json.Unmarshal(row.Data, apiComment)); err != nil { + return nil, err + } + // The owning issue id is carried in the raw row's input column. + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + comment := &models.LinearComment{ + ConnectionId: data.Options.ConnectionId, + Id: apiComment.Id, + IssueId: issueRef.OwningIssueId(), + Body: apiComment.Body, + CreatedAt: apiComment.CreatedAt, + UpdatedAt: apiComment.UpdatedAt, + } + if apiComment.User != nil { + comment.AuthorId = apiComment.User.Id + } + return []interface{}{comment}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_collector.go b/backend/plugins/linear/tasks/cycle_collector.go new file mode 100644 index 00000000000..88572650fcd --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_CYCLES_TABLE = "linear_cycles" + +// GraphqlQueryCycleWrapper is the team-scoped, paginated `cycles` query. +type GraphqlQueryCycleWrapper struct { + Team struct { + Cycles struct { + Nodes []GraphqlQueryCycle + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"cycles(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryCycle struct { + Id string + Number int + Name string + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time +} + +var CollectCyclesMeta = plugin.SubTaskMeta{ + Name: "Collect Cycles", + EntryPoint: CollectCycles, + EnabledByDefault: true, + Description: "Collect cycles (sprints) for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectCycles + +func CollectCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCycleWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCycleWrapper) + return query.Team.Cycles.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCycleWrapper) + for _, cycle := range query.Team.Cycles.Nodes { + messages = append(messages, errors.Must1(json.Marshal(cycle))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_convertor.go b/backend/plugins/linear/tasks/cycle_convertor.go new file mode 100644 index 00000000000..ae4a0dd5a84 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCyclesMeta = plugin.SubTaskMeta{ + Name: "Convert Cycles", + EntryPoint: ConvertCycles, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_cycles into domain layer tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearCycle{}.TableName(), RAW_CYCLES_TABLE}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertCycles + +func ConvertCycles(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearCycle{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearCycle{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + cycle := inputRow.(*models.LinearCycle) + sprintId := cycleIdGen.Generate(connectionId, cycle.Id) + name := cycle.Name + if name == "" { + name = fmt.Sprintf("Cycle %d", cycle.Number) + } + status := "ACTIVE" + if cycle.CompletedAt != nil { + status = "CLOSED" + } + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: name, + Status: status, + StartedDate: cycle.StartsAt, + EndedDate: cycle.EndsAt, + CompletedDate: cycle.CompletedAt, + OriginalBoardID: boardId, + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_extractor.go b/backend/plugins/linear/tasks/cycle_extractor.go new file mode 100644 index 00000000000..43a4233aa88 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_extractor.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCyclesMeta = plugin.SubTaskMeta{ + Name: "Extract Cycles", + EntryPoint: ExtractCycles, + EnabledByDefault: true, + Description: "Extract raw cycle data into tool layer table _tool_linear_cycles", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractCycles + +func ExtractCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiCycle := &GraphqlQueryCycle{} + if err := errors.Convert(json.Unmarshal(row.Data, apiCycle)); err != nil { + return nil, err + } + cycle := &models.LinearCycle{ + ConnectionId: data.Options.ConnectionId, + Id: apiCycle.Id, + TeamId: data.Options.TeamId, + Number: apiCycle.Number, + Name: apiCycle.Name, + StartsAt: apiCycle.StartsAt, + EndsAt: apiCycle.EndsAt, + CompletedAt: apiCycle.CompletedAt, + } + return []interface{}{cycle}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go new file mode 100644 index 00000000000..9f5a0aedd1d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUES_TABLE = "linear_issues" + +// GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. +// Incremental runs filter server-side on updatedAt ($filter) rather than +// relying on result ordering, so collection no longer depends on an undocumented +// default sort direction. +type GraphqlQueryIssueWrapper struct { + Team struct { + Issues struct { + Nodes []GraphqlQueryIssue `graphql:"nodes"` + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt, filter: $filter)"` + } `graphql:"team(id: $teamId)"` +} + +// IssueFilter mirrors the subset of Linear's GraphQL IssueFilter input used to +// restrict collection to issues updated after a point in time. The Go type +// name is significant: the GraphQL client emits it as the variable's type +// ($filter:IssueFilter!). +type IssueFilter struct { + UpdatedAt *DateComparator `json:"updatedAt,omitempty"` +} + +// DateComparator mirrors Linear's DateComparator input (only the `gt` operator +// is needed here). +type DateComparator struct { + Gt *time.Time `json:"gt,omitempty"` +} + +// buildIssueFilter returns an IssueFilter restricting to issues updated after +// `since`. When `since` is nil (a full sync) it returns the empty filter, which +// Linear treats as "match all". +func buildIssueFilter(since *time.Time) IssueFilter { + if since == nil { + return IssueFilter{} + } + return IssueFilter{UpdatedAt: &DateComparator{Gt: since}} +} + +type GraphqlQueryIssue struct { + Id string + Identifier string + Number int + Title string + Description string + Url string + Priority int + Estimate *float64 + CreatedAt time.Time + UpdatedAt time.Time + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + State *struct { + Id string + Name string + Type string + } + Assignee *struct{ Id string } + Creator *struct{ Id string } + Cycle *struct{ Id string } + Parent *struct{ Id string } + Labels struct { + Nodes []struct { + Id string + Name string + } + } `graphql:"labels(first: 50)"` +} + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "Collect Issues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "Collect issues for a Linear team, supports incremental collection", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssues + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }) + if err != nil { + return err + } + + since := apiCollector.GetSince() + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryIssueWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + "filter": buildIssueFilter(since), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryIssueWrapper) + return query.Team.Issues.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryIssueWrapper) + // The server-side $filter already restricts to issues updated after + // `since`, so every returned issue is in scope -- no client-side + // early-stop (and thus no dependency on sort direction) is needed. + for _, issue := range query.Team.Issues.Nodes { + issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) + issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) + issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) + messages = append(messages, errors.Must1(json.Marshal(issue))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector_test.go b/backend/plugins/linear/tasks/issue_collector_test.go new file mode 100644 index 00000000000..2dbbeecf586 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestBuildIssueFilter pins the server-side incremental filter that replaces +// the previous reliance on result ordering plus a client-side early-stop. A +// full sync must produce an empty filter (match all); an incremental run must +// produce Linear's IssueFilter shape `{ updatedAt: { gt: } }`. +func TestBuildIssueFilter(t *testing.T) { + full, err := json.Marshal(buildIssueFilter(nil)) + assert.NoError(t, err) + assert.Equal(t, "{}", string(full)) + + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental, err := json.Marshal(buildIssueFilter(&since)) + assert.NoError(t, err) + assert.JSONEq(t, `{"updatedAt":{"gt":"2026-05-01T00:00:00Z"}}`, string(incremental)) +} diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go new file mode 100644 index 00000000000..b124e59f369 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -0,0 +1,229 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Issues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssues + +func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + // Preload account display names so issues can carry assignee/creator names + // and emit issue_assignees rows, mirroring how the account convertor derives + // the domain account's full name (displayName, falling back to name). + var accounts []models.LinearAccount + if err := db.All(&accounts, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + accountNames := make(map[string]string, len(accounts)) + for _, account := range accounts { + name := account.Name + if account.DisplayName != "" { + name = account.DisplayName + } + accountNames[account.Id] = name + } + + // Preload issue labels so the convertor can derive domain issue Type from a + // label-based mapping (Linear has no native issue type). + var issueLabels []models.LinearIssueLabel + if err := db.All(&issueLabels, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + labelsByIssue := make(map[string][]string, len(issueLabels)) + for _, l := range issueLabels { + labelsByIssue[l.IssueId] = append(labelsByIssue[l.IssueId], l.LabelName) + } + + // Compile the label-matching patterns from the scope config. An empty + // pattern is treated as "no match" (nil), so issues default to REQUIREMENT. + typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) + if err != nil { + return err + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: typeMatcher.typeOf(labelsByIssue[issue.Id]), + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, + } + if issue.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + domainIssue.CreatorName = accountNames[issue.CreatorId] + } + if issue.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + domainIssue.AssigneeName = accountNames[issue.AssigneeId] + } + if issue.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) + domainIssue.IsSubtask = true + } + // Resolution date: completedAt, falling back to canceledAt. + if issue.CompletedAt != nil { + domainIssue.ResolutionDate = issue.CompletedAt + } else if issue.CanceledAt != nil { + domainIssue.ResolutionDate = issue.CanceledAt + } + // Fallback lead time when no history-derived value is present. + // Guard against a resolution that precedes creation (clock skew or + // migrated/imported issues): a negative duration cast to uint yields + // platform-dependent garbage, so leave lead time unset instead. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil && + domainIssue.ResolutionDate.After(issue.CreatedAt) { + minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// issueTypeMatcher derives the domain ticket.Issue.Type from an issue's label +// names using the scope config's regex patterns. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue whose labels match none (or with no +// patterns configured) defaults to REQUIREMENT. +type issueTypeMatcher struct { + incident *regexp.Regexp + bug *regexp.Regexp + requirement *regexp.Regexp +} + +func newIssueTypeMatcher(sc *models.LinearScopeConfig) (*issueTypeMatcher, errors.Error) { + m := &issueTypeMatcher{} + if sc == nil { + return m, nil + } + for _, p := range []struct { + pattern string + field string + out **regexp.Regexp + }{ + {sc.IssueTypeIncident, "issueTypeIncident", &m.incident}, + {sc.IssueTypeBug, "issueTypeBug", &m.bug}, + {sc.IssueTypeRequirement, "issueTypeRequirement", &m.requirement}, + } { + if p.pattern == "" { + continue + } + re, err := regexp.Compile(p.pattern) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid "+p.field+" pattern") + } + *p.out = re + } + return m, nil +} + +func (m *issueTypeMatcher) typeOf(labels []string) string { + for _, c := range []struct { + pattern *regexp.Regexp + typ string + }{ + {m.incident, ticket.INCIDENT}, + {m.bug, ticket.BUG}, + {m.requirement, ticket.REQUIREMENT}, + } { + if c.pattern == nil { + continue + } + for _, name := range labels { + if c.pattern.MatchString(name) { + return c.typ + } + } + } + return ticket.REQUIREMENT +} diff --git a/backend/plugins/linear/tasks/issue_extractor.go b/backend/plugins/linear/tasks/issue_extractor.go new file mode 100644 index 00000000000..e423076612d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_extractor.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "Extract Issues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "Extract raw issue data into tool layer tables _tool_linear_issues and _tool_linear_issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiIssue := &GraphqlQueryIssue{} + if err := errors.Convert(json.Unmarshal(row.Data, apiIssue)); err != nil { + return nil, err + } + connectionId := data.Options.ConnectionId + issue := &models.LinearIssue{ + ConnectionId: connectionId, + Id: apiIssue.Id, + TeamId: data.Options.TeamId, + Identifier: apiIssue.Identifier, + Number: apiIssue.Number, + Title: apiIssue.Title, + Description: apiIssue.Description, + Url: apiIssue.Url, + Priority: apiIssue.Priority, + PriorityLabel: PriorityLabel(apiIssue.Priority), + Estimate: apiIssue.Estimate, + CreatedAt: apiIssue.CreatedAt, + UpdatedAt: apiIssue.UpdatedAt, + StartedAt: apiIssue.StartedAt, + CompletedAt: apiIssue.CompletedAt, + CanceledAt: apiIssue.CanceledAt, + } + if apiIssue.State != nil { + issue.StateId = apiIssue.State.Id + issue.StateName = apiIssue.State.Name + issue.StateType = apiIssue.State.Type + } + if apiIssue.Assignee != nil { + issue.AssigneeId = apiIssue.Assignee.Id + } + if apiIssue.Creator != nil { + issue.CreatorId = apiIssue.Creator.Id + } + if apiIssue.Cycle != nil { + issue.CycleId = apiIssue.Cycle.Id + } + if apiIssue.Parent != nil { + issue.ParentId = apiIssue.Parent.Id + } + + results := make([]interface{}, 0, len(apiIssue.Labels.Nodes)+1) + results = append(results, issue) + for _, label := range apiIssue.Labels.Nodes { + results = append(results, &models.LinearIssueLabel{ + ConnectionId: connectionId, + IssueId: apiIssue.Id, + LabelName: label.Name, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go new file mode 100644 index 00000000000..11de7c218c7 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUE_HISTORY_TABLE = "linear_issue_history" + +// GraphqlQueryHistoryWrapper is the per-issue, paginated `history` query. +type GraphqlQueryHistoryWrapper struct { + Issue struct { + History struct { + Nodes []GraphqlQueryHistory + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"history(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryHistory struct { + Id string + CreatedAt time.Time + Actor *struct{ Id string } + FromState *struct { + Id string + Name string + Type string + } + ToState *struct { + Id string + Name string + Type string + } +} + +var CollectIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Collect Issue History", + EntryPoint: CollectIssueHistory, + EnabledByDefault: true, + Description: "Collect history events for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssueHistory + +func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's history cannot have changed, so re-fetching it every + // run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryHistoryWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryHistoryWrapper) + return query.Issue.History.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryHistoryWrapper) + for _, event := range query.Issue.History.Nodes { + // Only state transitions are relevant to the status changelog. + if event.FromState == nil && event.ToState == nil { + continue + } + messages = append(messages, errors.Must1(json.Marshal(event))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go new file mode 100644 index 00000000000..f1fa99d0519 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Convert Issue History", + EntryPoint: ConvertIssueHistory, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_history into domain layer table issue_changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueHistory{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUE_HISTORY_TABLE}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueHistory + +func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + historyIdGen := didgen.NewDomainIdGenerator(&models.LinearIssueHistory{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueHistory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + event := inputRow.(*models.LinearIssueHistory) + changelog := &ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{Id: historyIdGen.Generate(connectionId, event.Id)}, + IssueId: issueIdGen.Generate(connectionId, event.IssueId), + FieldId: "state", + FieldName: "status", + OriginalFromValue: event.FromStateName, + OriginalToValue: event.ToStateName, + CreatedDate: event.CreatedAt, + } + if event.FromStateType != "" { + changelog.FromValue = StatusFromStateType(event.FromStateType) + } + if event.ToStateType != "" { + changelog.ToValue = StatusFromStateType(event.ToStateType) + } + if event.ActorId != "" { + changelog.AuthorId = accountIdGen.Generate(connectionId, event.ActorId) + } + return []interface{}{changelog}, nil + }, + }) + if err != nil { + return err + } + if err := converter.Execute(); err != nil { + return err + } + + return deriveLeadTimeFromHistory(db, connectionId, data.Options.TeamId, issueIdGen) +} + +// deriveLeadTimeFromHistory refines each issue's lead time from its recorded +// state transitions: the span from the issue's first transition into an +// in-progress state to its first transition into a done state thereafter (the +// active cycle time). This is the value that genuinely requires history and is +// more accurate than the coarse createdAt -> resolutionDate fallback set by +// ConvertIssues, so it overrides that fallback when the transitions exist. +// Issues whose history lacks an in-progress -> done sequence keep the fallback. +func deriveLeadTimeFromHistory(db dal.Dal, connectionId uint64, teamId string, issueIdGen *didgen.DomainIdGenerator) errors.Error { + var events []models.LinearIssueHistory + if err := db.All(&events, + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, teamId), + dal.Orderby("h.issue_id, h.created_at"), + ); err != nil { + return err + } + + type leadWindow struct { + startedAt *time.Time + doneAt *time.Time + } + windows := map[string]*leadWindow{} + for i := range events { + event := events[i] + window := windows[event.IssueId] + if window == nil { + window = &leadWindow{} + windows[event.IssueId] = window + } + switch StatusFromStateType(event.ToStateType) { + case ticket.IN_PROGRESS: + if window.startedAt == nil { + createdAt := event.CreatedAt + window.startedAt = &createdAt + } + case ticket.DONE: + if window.startedAt != nil && window.doneAt == nil { + createdAt := event.CreatedAt + window.doneAt = &createdAt + } + } + } + + for issueId, window := range windows { + if window.startedAt == nil || window.doneAt == nil || !window.doneAt.After(*window.startedAt) { + continue + } + minutes := uint(window.doneAt.Sub(*window.startedAt).Minutes()) + if err := db.UpdateColumn( + &ticket.Issue{}, "lead_time_minutes", minutes, + dal.Where("id = ?", issueIdGen.Generate(connectionId, issueId)), + ); err != nil { + return err + } + } + return nil +} diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go new file mode 100644 index 00000000000..006b92f9701 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Extract Issue History", + EntryPoint: ExtractIssueHistory, + EnabledByDefault: true, + Description: "Extract raw issue history into tool layer table _tool_linear_issue_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssueHistory + +func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiEvent := &GraphqlQueryHistory{} + if err := errors.Convert(json.Unmarshal(row.Data, apiEvent)); err != nil { + return nil, err + } + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + event := &models.LinearIssueHistory{ + ConnectionId: data.Options.ConnectionId, + Id: apiEvent.Id, + IssueId: issueRef.OwningIssueId(), + CreatedAt: apiEvent.CreatedAt, + } + if apiEvent.Actor != nil { + event.ActorId = apiEvent.Actor.Id + } + if apiEvent.FromState != nil { + event.FromStateId = apiEvent.FromState.Id + event.FromStateName = apiEvent.FromState.Name + event.FromStateType = apiEvent.FromState.Type + } + if apiEvent.ToState != nil { + event.ToStateId = apiEvent.ToState.Id + event.ToStateName = apiEvent.ToState.Name + event.ToStateType = apiEvent.ToState.Type + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/label_convertor.go b/backend/plugins/linear/tasks/label_convertor.go new file mode 100644 index 00000000000..e94826d4775 --- /dev/null +++ b/backend/plugins/linear/tasks/label_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ + Name: "Convert Issue Labels", + EntryPoint: ConvertIssueLabels, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_labels into domain layer table issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueLabel{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.IssueLabel{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueLabels + +func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + + cursor, err := db.Cursor( + dal.Select("l.*"), + dal.From("_tool_linear_issue_labels l"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = l.connection_id AND i.id = l.issue_id)"), + dal.Where("l.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueLabel{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + label := inputRow.(*models.LinearIssueLabel) + domainLabel := &ticket.IssueLabel{ + IssueId: issueIdGen.Generate(connectionId, label.IssueId), + LabelName: label.LabelName, + } + return []interface{}{domainLabel}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go new file mode 100644 index 00000000000..0a8cfe9da7f --- /dev/null +++ b/backend/plugins/linear/tasks/shared.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// issuesToCollectChildrenClauses builds the cursor clauses that drive per-issue +// child collection (comments, history). When `since` is non-nil (an incremental +// run), it restricts the sweep to issues updated since the last successful +// collection, so unchanged issues no longer trigger a request every run. On a +// full sync `since` is nil and all of the team's issues are swept. +func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *time.Time) []dal.Clause { + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + } + if since != nil { + clauses = append(clauses, dal.Where("updated_at > ?", *since)) + } + return clauses +} + +// priorityLabels maps Linear's integer priority to its human-readable label. +// Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. +var priorityLabels = map[int]string{ + 0: "No priority", + 1: "Urgent", + 2: "High", + 3: "Medium", + 4: "Low", +} + +// PriorityLabel returns the human-readable label for a Linear priority value. +func PriorityLabel(priority int) string { + if label, ok := priorityLabels[priority]; ok { + return label + } + return "No priority" +} + +// StatusFromStateType maps a Linear WorkflowState.type to a DevLake standard +// issue status. Linear's state types are standardized, so no user-supplied +// mapping is required: +// +// triage, backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +// +// "triage" is the inbox state issues land in before they are accepted into a +// workflow; it is treated as not-yet-started (TODO). Any unrecognized type +// falls back to OTHER so unexpected API values surface rather than silently +// masquerading as a known status. +func StatusFromStateType(stateType string) string { + switch stateType { + case "triage", "backlog", "unstarted": + return ticket.TODO + case "started": + return ticket.IN_PROGRESS + case "completed", "canceled": + return ticket.DONE + default: + return ticket.OTHER + } +} diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go new file mode 100644 index 00000000000..ad2fc7bc6d0 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestIssuesToCollectChildrenClauses pins the incremental behaviour of the +// per-issue child collectors (comments, history): a full sync sweeps every +// issue, while an incremental run adds an updated_at filter so unchanged issues +// are skipped instead of triggering a request each run. +func TestIssuesToCollectChildrenClauses(t *testing.T) { + // full sync: no `since` -> select/from/where(connection,team) only + full := issuesToCollectChildrenClauses(1, "team-1", nil) + assert.Len(t, full, 3) + + // incremental: a `since` adds the updated_at filter clause + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental := issuesToCollectChildrenClauses(1, "team-1", &since) + assert.Len(t, incremental, 4) +} diff --git a/backend/plugins/linear/tasks/shared_test.go b/backend/plugins/linear/tasks/shared_test.go new file mode 100644 index 00000000000..eaf737c9022 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/stretchr/testify/assert" +) + +// TestStatusFromStateType pins the mapping for every Linear WorkflowState.type +// value. Linear's state types are standardized; "triage" is the inbox state +// issues land in before they are accepted, so it maps to TODO. Any genuinely +// unknown type falls back to OTHER. +func TestStatusFromStateType(t *testing.T) { + cases := map[string]string{ + "backlog": ticket.TODO, + "unstarted": ticket.TODO, + "triage": ticket.TODO, + "started": ticket.IN_PROGRESS, + "completed": ticket.DONE, + "canceled": ticket.DONE, + "": ticket.OTHER, + "something": ticket.OTHER, + } + for stateType, want := range cases { + assert.Equal(t, want, StatusFromStateType(stateType), "state type %q", stateType) + } +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go new file mode 100644 index 00000000000..7ebe467a908 --- /dev/null +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertSprintIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Sprint Issues", + EntryPoint: ConvertSprintIssues, + EnabledByDefault: true, + Description: "Link issues to their cycle (sprint) in the domain layer table sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.SprintIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprintIssues + +func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + + // Sprint membership is derived from each issue's cycle_id. Clear this team's + // existing sprint_issues up front so issues that have since left their cycle + // leave no stale rows: the batch divider only deletes outdated records when + // it produces at least one row of the type, which misses the case where + // every issue has been removed from its cycle. + var teamIssues []models.LinearIssue + if err := db.All(&teamIssues, + dal.Select("id"), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ); err != nil { + return err + } + if len(teamIssues) > 0 { + issueIds := make([]string, len(teamIssues)) + for i, issue := range teamIssues { + issueIds[i] = issueIdGen.Generate(connectionId, issue.Id) + } + if err := db.Delete(&ticket.SprintIssue{}, dal.Where("issue_id IN ?", issueIds)); err != nil { + return err + } + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + sprintIssue := &ticket.SprintIssue{ + SprintId: cycleIdGen.Generate(connectionId, issue.CycleId), + IssueId: issueIdGen.Generate(connectionId, issue.Id), + } + return []interface{}{sprintIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/task_data.go b/backend/plugins/linear/tasks/task_data.go new file mode 100644 index 00000000000..19f991d5f75 --- /dev/null +++ b/backend/plugins/linear/tasks/task_data.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearOptions are the per-scope options passed to a pipeline task. +type LinearOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// LinearTaskData is the shared context handed to every Linear subtask. +type LinearTaskData struct { + Options *LinearOptions + GraphqlClient *api.GraphqlAsyncClient + TimeAfter *time.Time + // ScopeConfig carries the resolved scope config (e.g. label-based issue-type + // mapping). Never nil: PrepareTaskData defaults it to an empty config. + ScopeConfig *models.LinearScopeConfig +} + +type LinearApiParams models.LinearApiParams diff --git a/backend/plugins/linear/tasks/workflow_state_collector.go b/backend/plugins/linear/tasks/workflow_state_collector.go new file mode 100644 index 00000000000..94c22d1d3a5 --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_WORKFLOW_STATES_TABLE = "linear_workflow_states" + +// GraphqlQueryWorkflowStateWrapper is the team-scoped paginated `states` query. +type GraphqlQueryWorkflowStateWrapper struct { + Team struct { + States struct { + Nodes []GraphqlQueryWorkflowState + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"states(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryWorkflowState struct { + Id string + Name string + Type string + Color string + Position float64 +} + +var CollectWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Collect Workflow States", + EntryPoint: CollectWorkflowStates, + EnabledByDefault: true, + Description: "Collect workflow states for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectWorkflowStates + +func CollectWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryWorkflowStateWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryWorkflowStateWrapper) + return query.Team.States.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryWorkflowStateWrapper) + for _, state := range query.Team.States.Nodes { + messages = append(messages, errors.Must1(json.Marshal(state))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/workflow_state_extractor.go b/backend/plugins/linear/tasks/workflow_state_extractor.go new file mode 100644 index 00000000000..75a2099aeaf --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_extractor.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Extract Workflow States", + EntryPoint: ExtractWorkflowStates, + EnabledByDefault: true, + Description: "Extract raw workflow state data into tool layer table _tool_linear_workflow_states", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractWorkflowStates + +func ExtractWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiState := &GraphqlQueryWorkflowState{} + if err := errors.Convert(json.Unmarshal(row.Data, apiState)); err != nil { + return nil, err + } + state := &models.LinearWorkflowState{ + ConnectionId: data.Options.ConnectionId, + Id: apiState.Id, + TeamId: data.Options.TeamId, + Name: apiState.Name, + Type: apiState.Type, + Color: apiState.Color, + Position: apiState.Position, + } + return []interface{}{state}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv index 8c288bf48cf..0955b9ccd93 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/board_issues.csv @@ -1,3 +1,4 @@ "board_id","issue_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" "github:GithubRepo:1:384111310","github:GithubIssue:1:1237324696","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,"" "github:GithubRepo:1:384111310","github:GithubIssue:1:1237324697","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,"" +"github:GithubRepo:1:384111310","github:GithubIssue:1:1237324698","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,"" diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv index 6b7d30fb8a3..d602f600d5e 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_request_issues.csv @@ -1,3 +1,4 @@ pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests," github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324697,7317,1885,,,0,"pull_requests," +github:GithubPullRequest:1:1819250574,github:GithubIssue:1:1237324698,7318,1886,,,0,"pull_requests," diff --git a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv index 22635f9b0b7..3084711b0bc 100644 --- a/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv +++ b/backend/plugins/linker/e2e/snapshot_tables/pull_requests.csv @@ -1,2 +1,3 @@ "id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date" -"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884 #1885","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" \ No newline at end of file +"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884 #1885","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" +"github:GithubPullRequest:1:1819250574","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",192,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#1886","14fb6488f2208e6a65374a86efce12dd460987e0","","MERGED","merged","","","Fix deployment policy recreation","no issue key here","https://github.com/apache/incubator-devlake/pull/7318","abeizn","github:GithubAccount:1:101256042","",7318,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000" diff --git a/backend/plugins/linker/impl/impl.go b/backend/plugins/linker/impl/impl.go index 917b22df9aa..e2efb809d9d 100644 --- a/backend/plugins/linker/impl/impl.go +++ b/backend/plugins/linker/impl/impl.go @@ -42,7 +42,7 @@ var _ interface { type Linker struct{} func (p Linker) Description() string { - return "link some cross table datas together" + return "link some cross table data together" } // RequiredDataEntities hasn't been used so far diff --git a/backend/plugins/linker/tasks/link_pr_and_issue.go b/backend/plugins/linker/tasks/link_pr_and_issue.go index 154cb4f311a..faeeffd716b 100644 --- a/backend/plugins/linker/tasks/link_pr_and_issue.go +++ b/backend/plugins/linker/tasks/link_pr_and_issue.go @@ -18,10 +18,11 @@ limitations under the License. package tasks import ( + "strings" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" - "strings" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -107,7 +108,7 @@ func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error { Enrich: func(pullRequest *code.PullRequest) ([]interface{}, errors.Error) { var issueKeys []string - for _, text := range []string{pullRequest.Title, pullRequest.Description} { + for _, text := range []string{pullRequest.Title, pullRequest.Description, pullRequest.HeadRef} { foundIssueKeys := data.PrToIssueRegexp.FindAllString(text, -1) if len(foundIssueKeys) > 0 { for _, issueKey := range foundIssueKeys { diff --git a/backend/plugins/opsgenie/api/connection_api.go b/backend/plugins/opsgenie/api/connection_api.go index c2844efaa1c..a56118cc73d 100644 --- a/backend/plugins/opsgenie/api/connection_api.go +++ b/backend/plugins/opsgenie/api/connection_api.go @@ -48,7 +48,7 @@ func testOpsgenieConn(ctx context.Context, connection models.OpsgenieConn) (*plu } if response.StatusCode == http.StatusForbidden { - return nil, errors.HttpStatus(http.StatusForbidden).New("API Key need 'Read' and 'Configuration access' Access rights") + return nil, errors.HttpStatus(http.StatusForbidden).New("API Key needs 'Read', 'Create/Update', and 'Configuration Access' access rights") } if response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted { diff --git a/backend/plugins/q_dev/api/blueprint_v200.go b/backend/plugins/q_dev/api/blueprint_v200.go index e3b845cb8ba..d7606a27564 100644 --- a/backend/plugins/q_dev/api/blueprint_v200.go +++ b/backend/plugins/q_dev/api/blueprint_v200.go @@ -72,6 +72,10 @@ func makeDataSourcePipelinePlanV200( ConnectionId: s3Slice.ConnectionId, S3Prefix: s3Slice.Prefix, ScopeId: s3Slice.Id, + AccountId: s3Slice.AccountId, + BasePath: s3Slice.BasePath, + Year: s3Slice.Year, + Month: s3Slice.Month, } // Pass empty entities array to enable all subtasks diff --git a/backend/plugins/q_dev/api/s3_slice_api.go b/backend/plugins/q_dev/api/s3_slice_api.go index 73708130294..158aa4ef785 100644 --- a/backend/plugins/q_dev/api/s3_slice_api.go +++ b/backend/plugins/q_dev/api/s3_slice_api.go @@ -60,62 +60,21 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er } // GetScope returns a single scope record -// @Summary get a Q Developer scope -// @Description get a Q Developer scope -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param blueprints query bool false "include blueprint references" -// @Success 200 {object} ScopeDetail -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [GET] func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.GetScopeDetail(input) } // PatchScope updates a scope record -// @Summary patch a Q Developer scope -// @Description patch a Q Developer scope -// @Tags plugins/q_dev -// @Accept application/json -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param scope body models.QDevS3Slice true "json" -// @Success 200 {object} models.QDevS3Slice -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [PATCH] func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.Patch(input) } // DeleteScope removes a scope and optionally associated data. -// @Summary delete a Q Developer scope -// @Description delete Q Developer scope data -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Param delete_data_only query bool false "Only delete scope data" -// @Success 200 -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId} [DELETE] func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.Delete(input) } // GetScopeLatestSyncState returns scope sync state info -// @Summary latest sync state for a Q Developer scope -// @Description get latest sync state for a Q Developer scope -// @Tags plugins/q_dev -// @Param connectionId path int true "connection ID" -// @Param scopeId path string true "scope id" -// @Success 200 -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/q_dev/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ScopeApi.GetScopeLatestSyncState(input) } diff --git a/backend/plugins/q_dev/impl/impl.go b/backend/plugins/q_dev/impl/impl.go index 80118212e10..3c6e1ed1644 100644 --- a/backend/plugins/q_dev/impl/impl.go +++ b/backend/plugins/q_dev/impl/impl.go @@ -57,6 +57,9 @@ func (p QDev) GetTablesInfo() []dal.Tabler { &models.QDevUserData{}, &models.QDevS3FileMeta{}, &models.QDevS3Slice{}, + &models.QDevUserReport{}, + &models.QDevChatLog{}, + &models.QDevCompletionLog{}, } } @@ -84,6 +87,7 @@ func (p QDev) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ tasks.CollectQDevS3FilesMeta, tasks.ExtractQDevS3DataMeta, + tasks.ExtractQDevLoggingDataMeta, } } @@ -117,10 +121,41 @@ func (p QDev) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int identityClient = nil } + // Resolve S3 prefixes to scan + var s3Prefixes []string + if op.AccountId != "" { + // New-style scope: construct both report paths using region from connection + region := connection.Region + timePart := fmt.Sprintf("%04d", op.Year) + if op.Month != nil { + timePart = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) + } + // Kiro exports data to two well-known S3 prefixes: + // {basePath}/AWSLogs/{accountId}/KiroLogs/ — user report CSVs + // logging/AWSLogs/{accountId}/KiroLogs/ — interaction logs (JSON.gz) + // When basePath is empty, default to "user-report" for CSV data. + reportBase := op.BasePath + if reportBase == "" { + reportBase = "user-report" + } + csvBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", reportBase, op.AccountId) + logBase := fmt.Sprintf("logging/AWSLogs/%s/KiroLogs", op.AccountId) + s3Prefixes = []string{ + fmt.Sprintf("%s/by_user_analytic/%s/%s", csvBase, region, timePart), + fmt.Sprintf("%s/user_report/%s/%s", csvBase, region, timePart), + fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", logBase, region, timePart), + fmt.Sprintf("%s/GenerateCompletions/%s/%s", logBase, region, timePart), + } + } else { + // Legacy scope: use S3Prefix directly + s3Prefixes = []string{op.S3Prefix} + } + return &tasks.QDevTaskData{ Options: &op, S3Client: s3Client, IdentityClient: identityClient, + S3Prefixes: s3Prefixes, }, nil } diff --git a/backend/plugins/q_dev/impl/impl_test.go b/backend/plugins/q_dev/impl/impl_test.go index 97dea86e622..7153617ab9a 100644 --- a/backend/plugins/q_dev/impl/impl_test.go +++ b/backend/plugins/q_dev/impl/impl_test.go @@ -34,11 +34,11 @@ func TestQDev_BasicPluginMethods(t *testing.T) { // Test table info tables := plugin.GetTablesInfo() - assert.Len(t, tables, 4) + assert.Len(t, tables, 7) // Test subtask metas subtasks := plugin.SubTaskMetas() - assert.Len(t, subtasks, 2) + assert.Len(t, subtasks, 3) // Test API resources apiResources := plugin.ApiResources() @@ -48,7 +48,7 @@ func TestQDev_BasicPluginMethods(t *testing.T) { } func TestQDev_TaskDataStructure(t *testing.T) { - // Test that QDevTaskData has the expected structure + // Test that QDevTaskData has the expected structure (legacy mode) taskData := &tasks.QDevTaskData{ Options: &tasks.QDevOptions{ ConnectionId: 1, @@ -61,6 +61,7 @@ func TestQDev_TaskDataStructure(t *testing.T) { StoreId: "d-1234567890", Region: "us-west-2", }, + S3Prefixes: []string{"test/"}, } assert.NotNil(t, taskData.Options) @@ -72,6 +73,36 @@ func TestQDev_TaskDataStructure(t *testing.T) { assert.Equal(t, "test-bucket", taskData.S3Client.Bucket) assert.Equal(t, "d-1234567890", taskData.IdentityClient.StoreId) assert.Equal(t, "us-west-2", taskData.IdentityClient.Region) + assert.Equal(t, []string{"test/"}, taskData.S3Prefixes) +} + +func TestQDev_TaskDataWithAccountId(t *testing.T) { + // Test new-style scope with AccountId and multiple S3Prefixes + month := 1 + taskData := &tasks.QDevTaskData{ + Options: &tasks.QDevOptions{ + ConnectionId: 1, + AccountId: "034362076319", + BasePath: "user-report", + Year: 2026, + Month: &month, + }, + S3Client: &tasks.QDevS3Client{ + Bucket: "test-bucket", + }, + S3Prefixes: []string{ + "user-report/AWSLogs/034362076319/KiroLogs/by_user_analytic/us-east-1/2026/01", + "user-report/AWSLogs/034362076319/KiroLogs/user_report/us-east-1/2026/01", + }, + } + + assert.Equal(t, "034362076319", taskData.Options.AccountId) + assert.Equal(t, "user-report", taskData.Options.BasePath) + assert.Equal(t, 2026, taskData.Options.Year) + assert.Equal(t, &month, taskData.Options.Month) + assert.Len(t, taskData.S3Prefixes, 2) + assert.Contains(t, taskData.S3Prefixes[0], "by_user_analytic") + assert.Contains(t, taskData.S3Prefixes[1], "user_report") } func TestQDev_TaskDataWithoutIdentityClient(t *testing.T) { @@ -83,10 +114,12 @@ func TestQDev_TaskDataWithoutIdentityClient(t *testing.T) { S3Client: &tasks.QDevS3Client{ Bucket: "test-bucket", }, - IdentityClient: nil, // No identity client + IdentityClient: nil, + S3Prefixes: []string{"some-prefix/"}, } assert.NotNil(t, taskData.Options) assert.NotNil(t, taskData.S3Client) assert.Nil(t, taskData.IdentityClient) + assert.Len(t, taskData.S3Prefixes, 1) } diff --git a/backend/plugins/q_dev/models/chat_log.go b/backend/plugins/q_dev/models/chat_log.go new file mode 100644 index 00000000000..6b39bffa4de --- /dev/null +++ b/backend/plugins/q_dev/models/chat_log.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevChatLog stores parsed data from GenerateAssistantResponse logging events +type QDevChatLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` + CodeReferenceCount int `json:"codeReferenceCount"` + WebLinkCount int `json:"webLinkCount"` + HasFollowupPrompts bool `json:"hasFollowupPrompts"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/completion_log.go b/backend/plugins/q_dev/models/completion_log.go new file mode 100644 index 00000000000..00a1b471f13 --- /dev/null +++ b/backend/plugins/q_dev/models/completion_log.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// QDevCompletionLog stores parsed data from GenerateCompletions logging events +type QDevCompletionLog struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` + LeftContextLength int `json:"leftContextLength"` + RightContextLength int `json:"rightContextLength"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go b/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go index e165cfd837c..f0c772faf91 100644 --- a/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go +++ b/backend/plugins/q_dev/models/migrationscripts/20250623_add_display_name_fields.go @@ -21,29 +21,44 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" ) var _ plugin.MigrationScript = (*addDisplayNameFields)(nil) type addDisplayNameFields struct{} -func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error { - db := basicRes.GetDal() +type QDevConnection20250623 struct { + IdentityStoreId string `gorm:"type:VARCHAR(255)"` + IdentityStoreRegion string `gorm:"type:VARCHAR(255)"` +} + +func (QDevConnection20250623) TableName() string { + return "_tool_q_dev_connections" +} + +type QDevUserData20250623 struct { + DisplayName string `gorm:"type:VARCHAR(255)"` +} - // Add Identity Center fields to connections table - // Ignore error if column already exists (MySQL error 1060) - _ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN identity_store_id VARCHAR(255)") - _ = db.Exec("ALTER TABLE _tool_q_dev_connections ADD COLUMN identity_store_region VARCHAR(255)") +func (QDevUserData20250623) TableName() string { + return "_tool_q_dev_user_data" +} - // Add display_name column to user_data table - // Ignore error if column already exists (MySQL error 1060) - _ = db.Exec("ALTER TABLE _tool_q_dev_user_data ADD COLUMN display_name VARCHAR(255)") +type QDevUserMetrics20250623 struct { + DisplayName string `gorm:"type:VARCHAR(255)"` +} - // Add display_name column to user_metrics table - // Ignore error if column already exists (MySQL error 1060) - _ = db.Exec("ALTER TABLE _tool_q_dev_user_metrics ADD COLUMN display_name VARCHAR(255)") +func (QDevUserMetrics20250623) TableName() string { + return "_tool_q_dev_user_metrics" +} - return nil +func (*addDisplayNameFields) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, + &QDevConnection20250623{}, + &QDevUserData20250623{}, + &QDevUserMetrics20250623{}, + ) } func (*addDisplayNameFields) Version() uint64 { diff --git a/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go index e76054d3aae..fdeb1058398 100644 --- a/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go +++ b/backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go @@ -20,23 +20,21 @@ package migrationscripts import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" ) type addScopeConfigIdToS3Slice struct{} +type QDevS3Slice20251123 struct { + ScopeConfigId uint64 `gorm:"type:BIGINT DEFAULT 0"` +} + +func (QDevS3Slice20251123) TableName() string { + return "_tool_q_dev_s3_slices" +} + func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error { - db := basicRes.GetDal() - - // Add scope_config_id column to _tool_q_dev_s3_slices table - err := db.Exec(` - ALTER TABLE _tool_q_dev_s3_slices - ADD COLUMN scope_config_id BIGINT DEFAULT 0 - `) - if err != nil { - return errors.Convert(err) - } - - return nil + return migrationhelper.AutoMigrateTables(basicRes, &QDevS3Slice20251123{}) } func (*addScopeConfigIdToS3Slice) Version() uint64 { diff --git a/backend/plugins/q_dev/models/migrationscripts/20251209_add_scope_id_fields.go b/backend/plugins/q_dev/models/migrationscripts/20251209_add_scope_id_fields.go index d7819ff6160..a4448b01248 100644 --- a/backend/plugins/q_dev/models/migrationscripts/20251209_add_scope_id_fields.go +++ b/backend/plugins/q_dev/models/migrationscripts/20251209_add_scope_id_fields.go @@ -19,6 +19,7 @@ package migrationscripts import ( "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" ) @@ -32,29 +33,24 @@ func (*addScopeIdFields) Up(basicRes context.BasicRes) errors.Error { // Add scope_id column to _tool_q_dev_user_data table // This field links user data to QDevS3Slice scope, which can then be mapped to projects via project_mapping - err := db.Exec(` - ALTER TABLE _tool_q_dev_user_data - ADD COLUMN IF NOT EXISTS scope_id VARCHAR(255) DEFAULT NULL - `) - if err != nil { - // Try alternative syntax for databases that don't support IF NOT EXISTS - _ = db.Exec(`ALTER TABLE _tool_q_dev_user_data ADD COLUMN scope_id VARCHAR(255) DEFAULT NULL`) + if !db.HasColumn("_tool_q_dev_user_data", "scope_id") { + if err := db.AddColumn("_tool_q_dev_user_data", "scope_id", dal.Varchar); err != nil { + return errors.Default.Wrap(err, "failed to add scope_id to _tool_q_dev_user_data") + } } // Add index on scope_id for better query performance - _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_q_dev_user_data_scope_id ON _tool_q_dev_user_data(scope_id)`) + _ = db.Exec(`CREATE INDEX idx_q_dev_user_data_scope_id ON _tool_q_dev_user_data(scope_id)`) // Add scope_id column to _tool_q_dev_s3_file_meta table - err = db.Exec(` - ALTER TABLE _tool_q_dev_s3_file_meta - ADD COLUMN IF NOT EXISTS scope_id VARCHAR(255) DEFAULT NULL - `) - if err != nil { - _ = db.Exec(`ALTER TABLE _tool_q_dev_s3_file_meta ADD COLUMN scope_id VARCHAR(255) DEFAULT NULL`) + if !db.HasColumn("_tool_q_dev_s3_file_meta", "scope_id") { + if err := db.AddColumn("_tool_q_dev_s3_file_meta", "scope_id", dal.Varchar); err != nil { + return errors.Default.Wrap(err, "failed to add scope_id to _tool_q_dev_s3_file_meta") + } } // Add index on scope_id - _ = db.Exec(`CREATE INDEX IF NOT EXISTS idx_q_dev_s3_file_meta_scope_id ON _tool_q_dev_s3_file_meta(scope_id)`) + _ = db.Exec(`CREATE INDEX idx_q_dev_s3_file_meta_scope_id ON _tool_q_dev_s3_file_meta(scope_id)`) return nil } diff --git a/backend/plugins/q_dev/models/migrationscripts/20260219_add_user_report_table.go b/backend/plugins/q_dev/models/migrationscripts/20260219_add_user_report_table.go new file mode 100644 index 00000000000..5f38c7407bf --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260219_add_user_report_table.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" +) + +type addUserReportTable struct{} + +func (*addUserReportTable) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevUserReport{}, + ) +} + +func (*addUserReportTable) Version() uint64 { + return 20260219000001 +} + +func (*addUserReportTable) Name() string { + return "Add user_report table for Kiro credits/subscription metrics" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260220_add_account_id_to_s3_slice.go b/backend/plugins/q_dev/models/migrationscripts/20260220_add_account_id_to_s3_slice.go new file mode 100644 index 00000000000..f0b0b897fa6 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260220_add_account_id_to_s3_slice.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*addAccountIdToS3Slice)(nil) + +type addAccountIdToS3Slice struct{} + +func (*addAccountIdToS3Slice) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + if !db.HasColumn("_tool_q_dev_s3_slices", "account_id") { + if err := db.AddColumn("_tool_q_dev_s3_slices", "account_id", dal.Varchar); err != nil { + return errors.Default.Wrap(err, "failed to add account_id to _tool_q_dev_s3_slices") + } + } + + return nil +} + +func (*addAccountIdToS3Slice) Version() uint64 { + return 20260220000001 +} + +func (*addAccountIdToS3Slice) Name() string { + return "add account_id column to _tool_q_dev_s3_slices for auto-constructing S3 prefixes" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260228_fix_dedup_user_tables.go b/backend/plugins/q_dev/models/migrationscripts/20260228_fix_dedup_user_tables.go new file mode 100644 index 00000000000..94b7666a9a5 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260228_fix_dedup_user_tables.go @@ -0,0 +1,60 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" +) + +type fixDedupUserTables struct{} + +func (*fixDedupUserTables) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + // Drop old tables that used auto-increment ID (which caused data duplication) + err := db.DropTables( + "_tool_q_dev_user_report", + "_tool_q_dev_user_data", + ) + if err != nil { + return errors.Default.Wrap(err, "failed to drop old user tables") + } + + // Recreate tables with composite primary keys for proper deduplication + err = migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevUserReportV2{}, + &archived.QDevUserDataV2{}, + ) + if err != nil { + return errors.Default.Wrap(err, "failed to recreate user tables") + } + + return nil +} + +func (*fixDedupUserTables) Version() uint64 { + return 20260228000001 +} + +func (*fixDedupUserTables) Name() string { + return "Rebuild user_report and user_data tables with composite primary keys to fix data duplication" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260228_reset_s3_file_meta_processed.go b/backend/plugins/q_dev/models/migrationscripts/20260228_reset_s3_file_meta_processed.go new file mode 100644 index 00000000000..7d0b8ba1f4f --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260228_reset_s3_file_meta_processed.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" +) + +type resetS3FileMetaProcessed struct{} + +func (*resetS3FileMetaProcessed) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + + // Reset processed flag so data will be re-extracted with the new + // dedup-safe composite-PK schema on next pipeline run + err := db.UpdateColumn( + "_tool_q_dev_s3_file_meta", + "processed", false, + dal.Where("1 = 1"), + ) + if err != nil { + return errors.Default.Wrap(err, "failed to reset s3_file_meta processed flag") + } + + return nil +} + +func (*resetS3FileMetaProcessed) Version() uint64 { + return 20260228000002 +} + +func (*resetS3FileMetaProcessed) Name() string { + return "Reset s3_file_meta processed flag to re-extract data with dedup-safe schema" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go new file mode 100644 index 00000000000..cbd5943ecd2 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260314_add_logging_tables.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" +) + +type addLoggingTables struct{} + +func (*addLoggingTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevChatLog{}, + &archived.QDevCompletionLog{}, + ) +} + +func (*addLoggingTables) Version() uint64 { + return 20260314000001 +} + +func (*addLoggingTables) Name() string { + return "Add chat_log and completion_log tables for Kiro logging data" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go new file mode 100644 index 00000000000..f98c3d1066f --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived" +) + +var _ = (*addLoggingFields)(nil) + +type addLoggingFields struct{} + +func (*addLoggingFields) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.QDevChatLog{}, + &archived.QDevCompletionLog{}, + ) +} + +func (*addLoggingFields) Version() uint64 { + return 20260319000001 +} + +func (*addLoggingFields) Name() string { + return "Add code_reference_count, web_link_count, has_followup_prompts to chat_log; left/right_context_length to completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go new file mode 100644 index 00000000000..ee7d10a1e87 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevChatLog struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + ChatTriggerType string `gorm:"type:varchar(50)" json:"chatTriggerType"` + HasCustomization bool `json:"hasCustomization"` + ConversationId string `gorm:"type:varchar(255)" json:"conversationId"` + UtteranceId string `gorm:"type:varchar(255)" json:"utteranceId"` + ModelId string `gorm:"type:varchar(100)" json:"modelId"` + PromptLength int `json:"promptLength"` + ResponseLength int `json:"responseLength"` + OpenFileCount int `json:"openFileCount"` + ActiveFileName string `gorm:"type:varchar(512)" json:"activeFileName"` + ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"` + HasSteering bool `json:"hasSteering"` + IsSpecMode bool `json:"isSpecMode"` + CodeReferenceCount int `json:"codeReferenceCount"` + WebLinkCount int `json:"webLinkCount"` + HasFollowupPrompts bool `json:"hasFollowupPrompts"` +} + +func (QDevChatLog) TableName() string { + return "_tool_q_dev_chat_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go new file mode 100644 index 00000000000..4acff956978 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevCompletionLog struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"` + UserId string `gorm:"index;type:varchar(255)" json:"userId"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Timestamp time.Time `gorm:"index" json:"timestamp"` + FileName string `gorm:"type:varchar(512)" json:"fileName"` + FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` + HasCustomization bool `json:"hasCustomization"` + CompletionsCount int `json:"completionsCount"` + LeftContextLength int `json:"leftContextLength"` + RightContextLength int `json:"rightContextLength"` +} + +func (QDevCompletionLog) TableName() string { + return "_tool_q_dev_completion_log" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/user_data_v2.go b/backend/plugins/q_dev/models/migrationscripts/archived/user_data_v2.go new file mode 100644 index 00000000000..8d5db8496a3 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/user_data_v2.go @@ -0,0 +1,81 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevUserDataV2 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + CodeReview_FindingsCount int + CodeReview_SucceededEventCount int + InlineChat_AcceptanceEventCount int + InlineChat_AcceptedLineAdditions int + InlineChat_AcceptedLineDeletions int + InlineChat_DismissalEventCount int + InlineChat_DismissedLineAdditions int + InlineChat_DismissedLineDeletions int + InlineChat_RejectedLineAdditions int + InlineChat_RejectedLineDeletions int + InlineChat_RejectionEventCount int + InlineChat_TotalEventCount int + Inline_AICodeLines int + Inline_AcceptanceCount int + Inline_SuggestionsCount int + Chat_AICodeLines int + Chat_MessagesInteracted int + Chat_MessagesSent int + CodeFix_AcceptanceEventCount int + CodeFix_AcceptedLines int + CodeFix_GeneratedLines int + CodeFix_GenerationEventCount int + CodeReview_FailedEventCount int + Dev_AcceptanceEventCount int + Dev_AcceptedLines int + Dev_GeneratedLines int + Dev_GenerationEventCount int + DocGeneration_AcceptedFileUpdates int + DocGeneration_AcceptedFilesCreations int + DocGeneration_AcceptedLineAdditions int + DocGeneration_AcceptedLineUpdates int + DocGeneration_EventCount int + DocGeneration_RejectedFileCreations int + DocGeneration_RejectedFileUpdates int + DocGeneration_RejectedLineAdditions int + DocGeneration_RejectedLineUpdates int + TestGeneration_AcceptedLines int + TestGeneration_AcceptedTests int + TestGeneration_EventCount int + TestGeneration_GeneratedLines int + TestGeneration_GeneratedTests int + Transformation_EventCount int + Transformation_LinesGenerated int + Transformation_LinesIngested int +} + +func (QDevUserDataV2) TableName() string { + return "_tool_q_dev_user_data" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/user_report.go b/backend/plugins/q_dev/models/migrationscripts/archived/user_report.go new file mode 100644 index 00000000000..53bef49b153 --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/user_report.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevUserReport struct { + archived.Model + ConnectionId uint64 `gorm:"primaryKey"` + UserId string `gorm:"index" json:"userId"` + Date time.Time `gorm:"index" json:"date"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + ScopeId string `gorm:"index;type:varchar(255)" json:"scopeId"` + ClientType string `gorm:"type:varchar(50)" json:"clientType"` + SubscriptionTier string `gorm:"type:varchar(50)" json:"subscriptionTier"` + ProfileId string `gorm:"type:varchar(512)" json:"profileId"` + ChatConversations int `json:"chatConversations"` + CreditsUsed float64 `json:"creditsUsed"` + OverageCap float64 `json:"overageCap"` + OverageCreditsUsed float64 `json:"overageCreditsUsed"` + OverageEnabled bool `json:"overageEnabled"` + TotalMessages int `json:"totalMessages"` +} + +func (QDevUserReport) TableName() string { + return "_tool_q_dev_user_report" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/archived/user_report_v2.go b/backend/plugins/q_dev/models/migrationscripts/archived/user_report_v2.go new file mode 100644 index 00000000000..7045874851c --- /dev/null +++ b/backend/plugins/q_dev/models/migrationscripts/archived/user_report_v2.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type QDevUserReportV2 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + ClientType string `gorm:"primaryKey;type:varchar(50)" json:"clientType"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + SubscriptionTier string `gorm:"type:varchar(50)" json:"subscriptionTier"` + ProfileId string `gorm:"type:varchar(512)" json:"profileId"` + ChatConversations int `json:"chatConversations"` + CreditsUsed float64 `json:"creditsUsed"` + OverageCap float64 `json:"overageCap"` + OverageCreditsUsed float64 `json:"overageCreditsUsed"` + OverageEnabled bool `json:"overageEnabled"` + TotalMessages int `json:"totalMessages"` +} + +func (QDevUserReportV2) TableName() string { + return "_tool_q_dev_user_report" +} diff --git a/backend/plugins/q_dev/models/migrationscripts/register.go b/backend/plugins/q_dev/models/migrationscripts/register.go index 86971e53931..8b5de0bcc16 100644 --- a/backend/plugins/q_dev/models/migrationscripts/register.go +++ b/backend/plugins/q_dev/models/migrationscripts/register.go @@ -31,5 +31,11 @@ func All() []plugin.MigrationScript { new(addS3SliceTable), new(addScopeConfigIdToS3Slice), new(addScopeIdFields), + new(addUserReportTable), + new(addAccountIdToS3Slice), + new(fixDedupUserTables), + new(resetS3FileMetaProcessed), + new(addLoggingTables), + new(addLoggingFields), } } diff --git a/backend/plugins/q_dev/models/s3_slice.go b/backend/plugins/q_dev/models/s3_slice.go index c844d16347f..19ecd6920bd 100644 --- a/backend/plugins/q_dev/models/s3_slice.go +++ b/backend/plugins/q_dev/models/s3_slice.go @@ -33,6 +33,7 @@ type QDevS3Slice struct { Id string `json:"id" mapstructure:"id" gorm:"primaryKey;type:varchar(512)"` Prefix string `json:"prefix" mapstructure:"prefix" gorm:"type:varchar(512);not null"` BasePath string `json:"basePath" mapstructure:"basePath" gorm:"type:varchar(512)"` + AccountId string `json:"accountId,omitempty" mapstructure:"accountId" gorm:"type:varchar(255)"` Year int `json:"year" mapstructure:"year" gorm:"not null"` Month *int `json:"month,omitempty" mapstructure:"month"` @@ -61,6 +62,7 @@ func (s *QDevS3Slice) normalize(strict bool) error { } s.BasePath = cleanPath(s.BasePath) + s.AccountId = strings.TrimSpace(s.AccountId) s.Prefix = cleanPath(selectNonEmpty(s.Prefix, s.Id)) if s.Year <= 0 { @@ -81,23 +83,46 @@ func (s *QDevS3Slice) normalize(strict bool) error { } } - if s.Prefix == "" { - s.Prefix = buildPrefix(s.BasePath, s.Year, s.Month) - } + if s.AccountId != "" { + // New-style scope: construct a logical identifier from component parts + s.Prefix = buildPrefixWithAccount(s.BasePath, s.AccountId, s.Year, s.Month) + } else { + // Legacy scope: derive prefix from basePath + year + month + if s.Prefix == "" { + s.Prefix = buildPrefix(s.BasePath, s.Year, s.Month) + } - prefix := buildPrefix(s.BasePath, s.Year, s.Month) - if prefix != "" { - s.Prefix = prefix + prefix := buildPrefix(s.BasePath, s.Year, s.Month) + if prefix != "" { + s.Prefix = prefix + } } if s.Id == "" { - s.Id = s.Prefix + if s.AccountId != "" { + // Use URL-safe ID: account_year or account_year_month + if s.Month != nil { + s.Id = fmt.Sprintf("%s_%04d_%02d", s.AccountId, s.Year, *s.Month) + } else { + s.Id = fmt.Sprintf("%s_%04d", s.AccountId, s.Year) + } + } else { + s.Id = s.Prefix + } } - if s.Month != nil { - s.Name = fmt.Sprintf("%04d-%02d", s.Year, *s.Month) - } else if s.Year > 0 { - s.Name = fmt.Sprintf("%04d", s.Year) + if s.AccountId != "" { + if s.Month != nil { + s.Name = fmt.Sprintf("%s %04d-%02d", s.AccountId, s.Year, *s.Month) + } else if s.Year > 0 { + s.Name = fmt.Sprintf("%s %04d", s.AccountId, s.Year) + } + } else { + if s.Month != nil { + s.Name = fmt.Sprintf("%04d-%02d", s.Year, *s.Month) + } else if s.Year > 0 { + s.Name = fmt.Sprintf("%04d", s.Year) + } } if s.FullName == "" { @@ -150,6 +175,14 @@ func (s QDevS3Slice) ScopeName() string { if s.Name != "" { return s.Name } + if s.AccountId != "" { + if s.Month != nil { + return fmt.Sprintf("%s %04d-%02d", s.AccountId, s.Year, *s.Month) + } + if s.Year > 0 { + return fmt.Sprintf("%s %04d", s.AccountId, s.Year) + } + } if s.Month != nil { return fmt.Sprintf("%04d-%02d", s.Year, *s.Month) } @@ -186,6 +219,20 @@ type QDevS3SliceParams struct { var _ plugin.ToolLayerScope = (*QDevS3Slice)(nil) +func buildPrefixWithAccount(basePath string, accountId string, year int, month *int) string { + parts := splitPath(basePath) + if accountId != "" { + parts = append(parts, accountId) + } + if year > 0 { + parts = append(parts, fmt.Sprintf("%04d", year)) + } + if month != nil { + parts = append(parts, fmt.Sprintf("%02d", *month)) + } + return strings.Join(parts, "/") +} + func buildPrefix(basePath string, year int, month *int) string { parts := splitPath(basePath) if year > 0 { diff --git a/backend/plugins/q_dev/models/user_data.go b/backend/plugins/q_dev/models/user_data.go index a668db78bdd..3d59f965ade 100644 --- a/backend/plugins/q_dev/models/user_data.go +++ b/backend/plugins/q_dev/models/user_data.go @@ -25,12 +25,12 @@ import ( // QDevUserData 存储从CSV中提取的原始数据 type QDevUserData struct { - common.Model + common.NoPKModel ConnectionId uint64 `gorm:"primaryKey"` - UserId string `gorm:"index" json:"userId"` - Date time.Time `gorm:"index" json:"date"` - DisplayName string `gorm:"type:varchar(255)" json:"displayName"` // New field for user display name - ScopeId string `gorm:"index;type:varchar(255)" json:"scopeId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` CodeReview_FindingsCount int CodeReview_SucceededEventCount int diff --git a/backend/plugins/q_dev/models/user_report.go b/backend/plugins/q_dev/models/user_report.go new file mode 100644 index 00000000000..17c4ac07daa --- /dev/null +++ b/backend/plugins/q_dev/models/user_report.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type QDevUserReport struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + UserId string `gorm:"primaryKey;type:varchar(255)" json:"userId"` + Date time.Time `gorm:"primaryKey;type:date" json:"date"` + ClientType string `gorm:"primaryKey;type:varchar(50)" json:"clientType"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + SubscriptionTier string `gorm:"type:varchar(50)" json:"subscriptionTier"` + ProfileId string `gorm:"type:varchar(512)" json:"profileId"` + ChatConversations int `json:"chatConversations"` + CreditsUsed float64 `json:"creditsUsed"` + OverageCap float64 `json:"overageCap"` + OverageCreditsUsed float64 `json:"overageCreditsUsed"` + OverageEnabled bool `json:"overageEnabled"` + TotalMessages int `json:"totalMessages"` +} + +func (QDevUserReport) TableName() string { + return "_tool_q_dev_user_report" +} diff --git a/backend/plugins/q_dev/tasks/s3_data_extractor.go b/backend/plugins/q_dev/tasks/s3_data_extractor.go index 03f8a6f5f85..da29bca07fe 100644 --- a/backend/plugins/q_dev/tasks/s3_data_extractor.go +++ b/backend/plugins/q_dev/tasks/s3_data_extractor.go @@ -40,10 +40,11 @@ func ExtractQDevS3Data(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*QDevTaskData) db := taskCtx.GetDal() - // 查询未处理的文件元数据 + // 查询未处理的CSV文件元数据(排除.json.gz日志文件) cursor, err := db.Cursor( dal.From(&models.QDevS3FileMeta{}), - dal.Where("connection_id = ? AND processed = ?", data.Options.ConnectionId, false), + dal.Where("connection_id = ? AND processed = ? AND file_name LIKE ?", + data.Options.ConnectionId, false, "%.csv"), ) if err != nil { return errors.Default.Wrap(err, "failed to get file metadata cursor") @@ -124,6 +125,14 @@ func processCSVData(taskCtx plugin.SubTaskContext, db dal.Dal, reader io.ReadClo return errors.Convert(err) } + // Auto-detect CSV format from headers + isNewFormat := detectUserReportFormat(headers) + if isNewFormat { + taskCtx.GetLogger().Debug("Detected new user_report CSV format") + } else { + taskCtx.GetLogger().Debug("Detected old by_user_analytic CSV format") + } + // 逐行读取数据 for { record, err := csvReader.Read() @@ -134,22 +143,134 @@ func processCSVData(taskCtx plugin.SubTaskContext, db dal.Dal, reader io.ReadClo return errors.Convert(err) } - // 创建用户数据对象 (updated to include display name resolution) - userData, err := createUserDataWithDisplayName(taskCtx.GetLogger(), headers, record, fileMeta, data.IdentityClient) - if err != nil { - return errors.Default.Wrap(err, "failed to create user data") - } + if isNewFormat { + reportData, err := createUserReportData(taskCtx.GetLogger(), headers, record, fileMeta, data.IdentityClient) + if err != nil { + return errors.Default.Wrap(err, "failed to create user report data") + } + err = db.CreateOrUpdate(reportData) + if err != nil { + return errors.Default.Wrap(err, "failed to save user report data") + } + } else { + // 创建用户数据对象 (updated to include display name resolution) + userData, err := createUserDataWithDisplayName(taskCtx.GetLogger(), headers, record, fileMeta, data.IdentityClient) + if err != nil { + return errors.Default.Wrap(err, "failed to create user data") + } - // Save to database - no need to check for duplicates since we're processing each file only once - err = db.Create(userData) - if err != nil { - return errors.Default.Wrap(err, "failed to save user data") + err = db.CreateOrUpdate(userData) + if err != nil { + return errors.Default.Wrap(err, "failed to save user data") + } } } return nil } +// detectUserReportFormat checks CSV headers to determine if this is the new user_report format +func detectUserReportFormat(headers []string) bool { + for _, h := range headers { + trimmed := strings.TrimSpace(h) + if trimmed == "Client_Type" || trimmed == "Credits_Used" { + return true + } + } + return false +} + +// createUserReportData creates a QDevUserReport from a new-format CSV record +func createUserReportData(logger interface { + Debug(format string, a ...interface{}) +}, headers []string, record []string, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver) (*models.QDevUserReport, errors.Error) { + report := &models.QDevUserReport{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + } + + // Build field map + fieldMap := make(map[string]string) + for i, header := range headers { + if i < len(record) { + logger.Debug("Mapping header[%d]: '%s' -> '%s'", i, header, record[i]) + fieldMap[header] = record[i] + trimmedHeader := strings.TrimSpace(header) + if trimmedHeader != header { + logger.Debug("Also adding trimmed header: '%s'", trimmedHeader) + fieldMap[trimmedHeader] = record[i] + } + } + } + + // UserId (normalize to strip "d-{directoryId}." prefix if present) + report.UserId = normalizeUserId(getStringField(fieldMap, "UserId")) + if report.UserId == "" { + return nil, errors.Default.New("UserId not found in CSV record") + } + + // DisplayName + report.DisplayName = resolveDisplayName(logger, report.UserId, identityClient) + + // Date + dateStr := getStringField(fieldMap, "Date") + if dateStr == "" { + return nil, errors.Default.New("Date not found in CSV record") + } + var err error + report.Date, err = parseDate(dateStr) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to parse date") + } + + // String fields + report.ClientType = getStringField(fieldMap, "Client_Type") + report.SubscriptionTier = getStringField(fieldMap, "Subscription_Tier") + report.ProfileId = getStringField(fieldMap, "ProfileId") + + // Numeric fields + report.ChatConversations = parseInt(fieldMap, "Chat_Conversations") + report.CreditsUsed = parseFloat(fieldMap, "Credits_Used") + report.OverageCap = parseFloat(fieldMap, "Overage_Cap") + report.OverageCreditsUsed = parseFloat(fieldMap, "Overage_Credits_Used") + report.OverageEnabled = parseBool(fieldMap, "Overage_Enabled") + report.TotalMessages = parseInt(fieldMap, "Total_Messages") + + return report, nil +} + +// getStringField returns the string value for a field, or empty string if not found +func getStringField(fieldMap map[string]string, field string) string { + value, ok := fieldMap[field] + if !ok { + return "" + } + return value +} + +// parseFloat extracts a float64 from the field map, returning 0 if missing or invalid +func parseFloat(fieldMap map[string]string, field string) float64 { + value, ok := fieldMap[field] + if !ok { + return 0 + } + f, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil { + return 0 + } + return f +} + +// parseBool extracts a boolean from the field map, returning false if missing or invalid +func parseBool(fieldMap map[string]string, field string) bool { + value, ok := fieldMap[field] + if !ok { + return false + } + trimmed := strings.TrimSpace(strings.ToLower(value)) + return trimmed == "true" || trimmed == "1" || trimmed == "yes" +} + // UserDisplayNameResolver interface for resolving user display names type UserDisplayNameResolver interface { ResolveUserDisplayName(userId string) (string, error) @@ -183,11 +304,12 @@ func createUserDataWithDisplayName(logger interface { var err error var ok bool - // 设置UserId - userData.UserId, ok = fieldMap["UserId"] + // 设置UserId (normalize to strip "d-{directoryId}." prefix if present) + rawUserId, ok := fieldMap["UserId"] if !ok { return nil, errors.Default.New("UserId not found in CSV record") } + userData.UserId = normalizeUserId(rawUserId) // 设置DisplayName (new functionality) userData.DisplayName = resolveDisplayName(logger, userData.UserId, identityClient) diff --git a/backend/plugins/q_dev/tasks/s3_data_extractor_test.go b/backend/plugins/q_dev/tasks/s3_data_extractor_test.go index fbea299dc28..0a5f808ebb1 100644 --- a/backend/plugins/q_dev/tasks/s3_data_extractor_test.go +++ b/backend/plugins/q_dev/tasks/s3_data_extractor_test.go @@ -419,6 +419,173 @@ func TestParseDate(t *testing.T) { } } +func TestDetectUserReportFormat(t *testing.T) { + // New format: contains Client_Type + assert.True(t, detectUserReportFormat([]string{"UserId", "Date", "Client_Type", "Credits_Used"})) + // New format: contains Credits_Used + assert.True(t, detectUserReportFormat([]string{"UserId", "Date", "Credits_Used", "Total_Messages"})) + // Old format: code-level metrics + assert.False(t, detectUserReportFormat([]string{"UserId", "Date", "Chat_AICodeLines", "Inline_AICodeLines"})) + // Old format: no new-format indicators + assert.False(t, detectUserReportFormat([]string{"UserId", "Date", "CodeReview_FindingsCount"})) + // Empty headers + assert.False(t, detectUserReportFormat([]string{})) + // Whitespace-padded header still detected + assert.True(t, detectUserReportFormat([]string{"UserId", " Client_Type ", "Date"})) +} + +func TestCreateUserReportData_Success(t *testing.T) { + headers := []string{ + "UserId", "Date", "Client_Type", "Subscription_Tier", "ProfileId", + "Chat_Conversations", "Credits_Used", "Overage_Cap", "Overage_Credits_Used", + "Overage_Enabled", "Total_Messages", + } + record := []string{ + "user-abc", "2026-01-15", "KIRO_IDE", "Pro", "profile-xyz", + "12", "45.5", "100.0", "5.25", + "true", "87", + } + fileMeta := &models.QDevS3FileMeta{ + ConnectionId: 1, + ScopeId: "scope-1", + } + + mockIdentityClient := &MockIdentityClient{} + mockIdentityClient.On("ResolveUserDisplayName", "user-abc").Return("Alice Bob", nil) + + mockLogger := &MockLogger{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + report, err := createUserReportData(mockLogger, headers, record, fileMeta, mockIdentityClient) + + assert.NoError(t, err) + assert.NotNil(t, report) + assert.Equal(t, "user-abc", report.UserId) + assert.Equal(t, "Alice Bob", report.DisplayName) + assert.Equal(t, uint64(1), report.ConnectionId) + assert.Equal(t, "scope-1", report.ScopeId) + assert.Equal(t, "KIRO_IDE", report.ClientType) + assert.Equal(t, "Pro", report.SubscriptionTier) + assert.Equal(t, "profile-xyz", report.ProfileId) + assert.Equal(t, 12, report.ChatConversations) + assert.Equal(t, 45.5, report.CreditsUsed) + assert.Equal(t, 100.0, report.OverageCap) + assert.Equal(t, 5.25, report.OverageCreditsUsed) + assert.True(t, report.OverageEnabled) + assert.Equal(t, 87, report.TotalMessages) + + expectedDate, _ := time.Parse("2006-01-02", "2026-01-15") + assert.Equal(t, expectedDate, report.Date) + + mockIdentityClient.AssertExpectations(t) +} + +func TestCreateUserReportData_MissingUserId(t *testing.T) { + headers := []string{"Date", "Client_Type", "Credits_Used"} + record := []string{"2026-01-15", "KIRO_IDE", "10.0"} + fileMeta := &models.QDevS3FileMeta{ConnectionId: 1} + + mockLogger := &MockLogger{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + report, err := createUserReportData(mockLogger, headers, record, fileMeta, nil) + + assert.Error(t, err) + assert.Nil(t, report) + assert.Contains(t, err.Error(), "UserId not found") +} + +func TestCreateUserReportData_MissingDate(t *testing.T) { + headers := []string{"UserId", "Client_Type", "Credits_Used"} + record := []string{"user-abc", "KIRO_IDE", "10.0"} + fileMeta := &models.QDevS3FileMeta{ConnectionId: 1} + + mockLogger := &MockLogger{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + report, err := createUserReportData(mockLogger, headers, record, fileMeta, nil) + + assert.Error(t, err) + assert.Nil(t, report) + assert.Contains(t, err.Error(), "Date not found") +} + +func TestCreateUserReportData_OverageDisabled(t *testing.T) { + headers := []string{"UserId", "Date", "Overage_Enabled", "Credits_Used"} + record := []string{"user-abc", "2026-01-15", "false", "10.0"} + fileMeta := &models.QDevS3FileMeta{ConnectionId: 1} + + mockLogger := &MockLogger{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + report, err := createUserReportData(mockLogger, headers, record, fileMeta, nil) + + assert.NoError(t, err) + assert.False(t, report.OverageEnabled) +} + +func TestCreateUserReportData_InvalidNumericValues(t *testing.T) { + headers := []string{"UserId", "Date", "Credits_Used", "Chat_Conversations", "Total_Messages"} + record := []string{"user-abc", "2026-01-15", "not-a-float", "not-an-int", ""} + fileMeta := &models.QDevS3FileMeta{ConnectionId: 1} + + mockLogger := &MockLogger{} + mockLogger.On("Debug", mock.Anything, mock.Anything).Return() + + report, err := createUserReportData(mockLogger, headers, record, fileMeta, nil) + + assert.NoError(t, err) + assert.Equal(t, float64(0), report.CreditsUsed) + assert.Equal(t, 0, report.ChatConversations) + assert.Equal(t, 0, report.TotalMessages) +} + +func TestParseFloat(t *testing.T) { + fieldMap := map[string]string{ + "ValidFloat": "3.14", + "ZeroFloat": "0", + "NegativeFloat": "-2.5", + "IntegerValue": "42", + "InvalidFloat": "not-a-number", + "EmptyString": "", + "Whitespace": " 1.5 ", + } + + assert.Equal(t, 3.14, parseFloat(fieldMap, "ValidFloat")) + assert.Equal(t, float64(0), parseFloat(fieldMap, "ZeroFloat")) + assert.Equal(t, -2.5, parseFloat(fieldMap, "NegativeFloat")) + assert.Equal(t, float64(42), parseFloat(fieldMap, "IntegerValue")) + assert.Equal(t, float64(0), parseFloat(fieldMap, "InvalidFloat")) + assert.Equal(t, float64(0), parseFloat(fieldMap, "EmptyString")) + assert.Equal(t, 1.5, parseFloat(fieldMap, "Whitespace")) + assert.Equal(t, float64(0), parseFloat(fieldMap, "NonExistentField")) +} + +func TestParseBool(t *testing.T) { + fieldMap := map[string]string{ + "TrueValue": "true", + "TrueUpper": "True", + "TrueOne": "1", + "TrueYes": "yes", + "FalseValue": "false", + "FalseZero": "0", + "EmptyString": "", + "InvalidBool": "maybe", + "WhitespaceVal": " true ", + } + + assert.True(t, parseBool(fieldMap, "TrueValue")) + assert.True(t, parseBool(fieldMap, "TrueUpper")) + assert.True(t, parseBool(fieldMap, "TrueOne")) + assert.True(t, parseBool(fieldMap, "TrueYes")) + assert.False(t, parseBool(fieldMap, "FalseValue")) + assert.False(t, parseBool(fieldMap, "FalseZero")) + assert.False(t, parseBool(fieldMap, "EmptyString")) + assert.False(t, parseBool(fieldMap, "InvalidBool")) + assert.True(t, parseBool(fieldMap, "WhitespaceVal")) + assert.False(t, parseBool(fieldMap, "NonExistentField")) +} + func TestParseInt(t *testing.T) { fieldMap := map[string]string{ "ValidInt": "42", diff --git a/backend/plugins/q_dev/tasks/s3_file_collector.go b/backend/plugins/q_dev/tasks/s3_file_collector.go index ae88fb97aa4..1ab4f8f0aa7 100644 --- a/backend/plugins/q_dev/tasks/s3_file_collector.go +++ b/backend/plugins/q_dev/tasks/s3_file_collector.go @@ -30,81 +30,79 @@ import ( var _ plugin.SubTaskEntryPoint = CollectQDevS3Files -// CollectQDevS3Files 收集S3文件元数据 +// CollectQDevS3Files collects S3 file metadata func CollectQDevS3Files(taskCtx plugin.SubTaskContext) errors.Error { data := taskCtx.GetData().(*QDevTaskData) db := taskCtx.GetDal() - // 列出指定前缀下的所有对象 - var continuationToken *string - prefix := data.Options.S3Prefix - if prefix != "" && !strings.HasSuffix(prefix, "/") { - prefix = prefix + "/" - } - taskCtx.SetProgress(0, -1) - for { - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(data.S3Client.Bucket), - Prefix: aws.String(prefix), - ContinuationToken: continuationToken, + for _, rawPrefix := range data.S3Prefixes { + prefix := rawPrefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix = prefix + "/" } - result, err := data.S3Client.S3.ListObjectsV2(input) - if err != nil { - return errors.Convert(err) - } + taskCtx.GetLogger().Info("Scanning S3 prefix: %s", prefix) - // 处理每个CSV文件 - for _, object := range result.Contents { - // Only process CSV files - if !strings.HasSuffix(*object.Key, ".csv") { - taskCtx.GetLogger().Debug("Skipping non-CSV file: %s", *object.Key) - continue + var continuationToken *string + for { + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(data.S3Client.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, } - // Check if this file already exists in our database - existingFile := &models.QDevS3FileMeta{} - err = db.First(existingFile, dal.Where("connection_id = ? AND s3_path = ?", - data.Options.ConnectionId, *object.Key)) + result, err := data.S3Client.S3.ListObjectsV2(input) + if err != nil { + return errors.Convert(err) + } - if err == nil { - // File already exists in database, skip it if it's already processed - if existingFile.Processed { - taskCtx.GetLogger().Debug("Skipping already processed file: %s", *object.Key) + for _, object := range result.Contents { + // Only process CSV and JSON.gz files + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + taskCtx.GetLogger().Debug("Skipping unsupported file: %s", *object.Key) continue } - // Otherwise, we'll keep the existing record (which is still marked as unprocessed) - taskCtx.GetLogger().Debug("Found existing unprocessed file: %s", *object.Key) - continue - } else if !db.IsErrorNotFound(err) { - return errors.Default.Wrap(err, "failed to query existing file metadata") - } - // This is a new file, save its metadata - fileMeta := &models.QDevS3FileMeta{ - ConnectionId: data.Options.ConnectionId, - FileName: *object.Key, - S3Path: *object.Key, - ScopeId: data.Options.ScopeId, - Processed: false, - } + // Check if this file already exists in our database + existingFile := &models.QDevS3FileMeta{} + err = db.First(existingFile, dal.Where("connection_id = ? AND s3_path = ?", + data.Options.ConnectionId, *object.Key)) + + if err == nil { + if existingFile.Processed { + taskCtx.GetLogger().Debug("Skipping already processed file: %s", *object.Key) + continue + } + taskCtx.GetLogger().Debug("Found existing unprocessed file: %s", *object.Key) + continue + } else if !db.IsErrorNotFound(err) { + return errors.Default.Wrap(err, "failed to query existing file metadata") + } - err = db.Create(fileMeta) - if err != nil { - return errors.Default.Wrap(err, "failed to create file metadata") + fileMeta := &models.QDevS3FileMeta{ + ConnectionId: data.Options.ConnectionId, + FileName: *object.Key, + S3Path: *object.Key, + ScopeId: data.Options.ScopeId, + Processed: false, + } + + err = db.Create(fileMeta) + if err != nil { + return errors.Default.Wrap(err, "failed to create file metadata") + } + + taskCtx.IncProgress(1) } - taskCtx.IncProgress(1) - } + if !*result.IsTruncated { + break + } - // 如果没有更多对象,退出循环 - if !*result.IsTruncated { - break + continuationToken = result.NextContinuationToken } - - continuationToken = result.NextContinuationToken } return nil diff --git a/backend/plugins/q_dev/tasks/s3_logging_extractor.go b/backend/plugins/q_dev/tasks/s3_logging_extractor.go new file mode 100644 index 00000000000..3fa771789a1 --- /dev/null +++ b/backend/plugins/q_dev/tasks/s3_logging_extractor.go @@ -0,0 +1,450 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "compress/gzip" + "encoding/json" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/q_dev/models" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +var _ plugin.SubTaskEntryPoint = ExtractQDevLoggingData + +const ( + loggingBatchSize = 50 // number of files to process per DB transaction + s3DownloadWorkers = 10 // parallel S3 download goroutines + s3DownloadChanSize = 20 // buffered channel size for download results +) + +// downloadResult holds the parsed records from one S3 file +type downloadResult struct { + FileMeta *models.QDevS3FileMeta + ChatLogs []*models.QDevChatLog + CompLogs []*models.QDevCompletionLog + Err error +} + +// ExtractQDevLoggingData extracts logging data from S3 JSON.gz files +func ExtractQDevLoggingData(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*QDevTaskData) + db := taskCtx.GetDal() + + cursor, err := db.Cursor( + dal.From(&models.QDevS3FileMeta{}), + dal.Where("connection_id = ? AND processed = ? AND file_name LIKE ?", + data.Options.ConnectionId, false, "%.json.gz"), + ) + if err != nil { + return errors.Default.Wrap(err, "failed to get logging file metadata cursor") + } + defer cursor.Close() + + // Collect all file metas first + var fileMetas []*models.QDevS3FileMeta + for cursor.Next() { + fm := &models.QDevS3FileMeta{} + if err := db.Fetch(cursor, fm); err != nil { + return errors.Default.Wrap(err, "failed to fetch file metadata") + } + fileMetas = append(fileMetas, fm) + } + + if len(fileMetas) == 0 { + return nil + } + + taskCtx.SetProgress(0, len(fileMetas)) + taskCtx.GetLogger().Info("Processing %d logging files with %d workers", len(fileMetas), s3DownloadWorkers) + + // Display name cache to avoid repeated IAM calls + displayNameCache := &sync.Map{} + + // Process in batches + for batchStart := 0; batchStart < len(fileMetas); batchStart += loggingBatchSize { + batchEnd := batchStart + loggingBatchSize + if batchEnd > len(fileMetas) { + batchEnd = len(fileMetas) + } + batch := fileMetas[batchStart:batchEnd] + + // Parallel download and parse + results := parallelDownloadAndParse(taskCtx, data, batch, displayNameCache) + + // Check for download errors + for _, r := range results { + if r.Err != nil { + return errors.Default.Wrap(errors.Convert(r.Err), + "failed to download/parse "+r.FileMeta.FileName) + } + } + + // Batch write to DB in a single transaction + tx := db.Begin() + var txErr errors.Error + for _, r := range results { + for _, chatLog := range r.ChatLogs { + if txErr = tx.CreateOrUpdate(chatLog); txErr != nil { + break + } + } + if txErr != nil { + break + } + for _, compLog := range r.CompLogs { + if txErr = tx.CreateOrUpdate(compLog); txErr != nil { + break + } + } + if txErr != nil { + break + } + r.FileMeta.Processed = true + now := time.Now() + r.FileMeta.ProcessedTime = &now + if txErr = tx.Update(r.FileMeta); txErr != nil { + break + } + } + if txErr != nil { + if rbErr := tx.Rollback(); rbErr != nil { + taskCtx.GetLogger().Error(rbErr, "failed to rollback transaction") + } + return errors.Default.Wrap(txErr, "failed to write logging batch") + } + if err := tx.Commit(); err != nil { + return errors.Default.Wrap(err, "failed to commit batch") + } + + taskCtx.IncProgress(len(batch)) + } + + return nil +} + +// parallelDownloadAndParse downloads and parses S3 files concurrently +func parallelDownloadAndParse( + taskCtx plugin.SubTaskContext, + data *QDevTaskData, + fileMetas []*models.QDevS3FileMeta, + displayNameCache *sync.Map, +) []downloadResult { + results := make([]downloadResult, len(fileMetas)) + jobs := make(chan int, s3DownloadChanSize) + var wg sync.WaitGroup + + // Start workers + for w := 0; w < s3DownloadWorkers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for idx := range jobs { + fm := fileMetas[idx] + result := downloadAndParseFile(taskCtx, data, fm, displayNameCache) + results[idx] = result + } + }() + } + + // Send jobs + for i := range fileMetas { + jobs <- i + } + close(jobs) + wg.Wait() + + return results +} + +// downloadAndParseFile downloads one S3 file and parses it into model records +func downloadAndParseFile( + taskCtx plugin.SubTaskContext, + data *QDevTaskData, + fileMeta *models.QDevS3FileMeta, + displayNameCache *sync.Map, +) downloadResult { + result := downloadResult{FileMeta: fileMeta} + + getResult, err := data.S3Client.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(data.S3Client.Bucket), + Key: aws.String(fileMeta.S3Path), + }) + if err != nil { + result.Err = err + return result + } + defer getResult.Body.Close() + + gzReader, err := gzip.NewReader(getResult.Body) + if err != nil { + result.Err = err + return result + } + defer gzReader.Close() + + var logFile loggingFile + if err := json.NewDecoder(gzReader).Decode(&logFile); err != nil { + result.Err = err + return result + } + + isChatLog := strings.Contains(fileMeta.S3Path, "GenerateAssistantResponse") + + for _, rawRecord := range logFile.Records { + if isChatLog { + chatLog, err := parseChatRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache) + if err != nil { + result.Err = err + return result + } + if chatLog != nil { + result.ChatLogs = append(result.ChatLogs, chatLog) + } + } else { + compLog, err := parseCompletionRecord(rawRecord, fileMeta, data.IdentityClient, displayNameCache) + if err != nil { + result.Err = err + return result + } + if compLog != nil { + result.CompLogs = append(result.CompLogs, compLog) + } + } + } + + return result +} + +// cachedResolveDisplayName resolves display name with caching +func cachedResolveDisplayName(userId string, identityClient UserDisplayNameResolver, cache *sync.Map) string { + if v, ok := cache.Load(userId); ok { + return v.(string) + } + if identityClient == nil { + cache.Store(userId, userId) + return userId + } + displayName, err := identityClient.ResolveUserDisplayName(userId) + if err != nil || displayName == "" { + cache.Store(userId, userId) + return userId + } + cache.Store(userId, displayName) + return displayName +} + +// JSON structures for logging data + +type loggingFile struct { + Records []json.RawMessage `json:"records"` +} + +type chatLogRecord struct { + Request *chatLogRequest `json:"generateAssistantResponseEventRequest"` + Response *chatLogResponse `json:"generateAssistantResponseEventResponse"` +} + +type chatLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + ChatTriggerType string `json:"chatTriggerType"` + CustomizationArn *string `json:"customizationArn"` + ModelID string `json:"modelId"` + Prompt string `json:"prompt"` +} + +type chatLogResponse struct { + RequestID string `json:"requestId"` + AssistantResponse string `json:"assistantResponse"` + FollowupPrompts string `json:"followupPrompts"` + MessageMetadata struct { + ConversationID *string `json:"conversationId"` + UtteranceID *string `json:"utteranceId"` + } `json:"messageMetadata"` + CodeReferenceEvents []json.RawMessage `json:"codeReferenceEvents"` + SupplementaryWebLinksEvent []json.RawMessage `json:"supplementaryWebLinksEvent"` +} + +type completionLogRecord struct { + Request *completionLogRequest `json:"generateCompletionsEventRequest"` + Response *completionLogResponse `json:"generateCompletionsEventResponse"` +} + +type completionLogRequest struct { + UserID string `json:"userId"` + Timestamp string `json:"timeStamp"` + FileName string `json:"fileName"` + CustomizationArn *string `json:"customizationArn"` + LeftContext string `json:"leftContext"` + RightContext string `json:"rightContext"` +} + +type completionLogResponse struct { + RequestID string `json:"requestId"` + Completions []json.RawMessage `json:"completions"` +} + +func parseChatRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver, cache *sync.Map) (*models.QDevChatLog, error) { + var record chatLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return nil, err + } + + if record.Request == nil || record.Response == nil { + return nil, nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + userId := normalizeUserId(record.Request.UserID) + chatLog := &models.QDevChatLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: userId, + DisplayName: cachedResolveDisplayName(userId, identityClient, cache), + Timestamp: ts, + ChatTriggerType: record.Request.ChatTriggerType, + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + ModelId: record.Request.ModelID, + PromptLength: len(record.Request.Prompt), + ResponseLength: len(record.Response.AssistantResponse), + } + + // Parse structured info from prompt + prompt := record.Request.Prompt + chatLog.OpenFileCount = countOpenFiles(prompt) + chatLog.ActiveFileName, chatLog.ActiveFileExtension = parseActiveFile(prompt) + chatLog.HasSteering = strings.Contains(prompt, ".kiro/steering") + chatLog.IsSpecMode = strings.Contains(prompt, "implicit-rules") + + if record.Response.MessageMetadata.ConversationID != nil { + chatLog.ConversationId = *record.Response.MessageMetadata.ConversationID + } + if record.Response.MessageMetadata.UtteranceID != nil { + chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID + } + + // New fields from docs: codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts + chatLog.CodeReferenceCount = len(record.Response.CodeReferenceEvents) + chatLog.WebLinkCount = len(record.Response.SupplementaryWebLinksEvent) + chatLog.HasFollowupPrompts = record.Response.FollowupPrompts != "" + + return chatLog, nil +} + +// countOpenFiles counts tags within block +func countOpenFiles(prompt string) int { + start := strings.Index(prompt, "") + if start == -1 { + return 0 + } + end := strings.Index(prompt, "") + if end == -1 { + return 0 + } + block := prompt[start:end] + return strings.Count(block, "") + if start == -1 { + return "", "" + } + end := strings.Index(prompt[start:], "") + if end == -1 { + return "", "" + } + block := prompt[start : start+end] + nameStart := strings.Index(block, "name=\"") + if nameStart == -1 { + return "", "" + } + nameStart += len("name=\"") + nameEnd := strings.Index(block[nameStart:], "\"") + if nameEnd == -1 { + return "", "" + } + fileName := block[nameStart : nameStart+nameEnd] + ext := filepath.Ext(fileName) + return fileName, ext +} + +func parseCompletionRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, identityClient UserDisplayNameResolver, cache *sync.Map) (*models.QDevCompletionLog, error) { + var record completionLogRecord + if err := json.Unmarshal(raw, &record); err != nil { + return nil, err + } + + if record.Request == nil || record.Response == nil { + return nil, nil + } + + ts, err := time.Parse(time.RFC3339Nano, record.Request.Timestamp) + if err != nil { + ts = time.Now() + } + + userId := normalizeUserId(record.Request.UserID) + return &models.QDevCompletionLog{ + ConnectionId: fileMeta.ConnectionId, + ScopeId: fileMeta.ScopeId, + RequestId: record.Response.RequestID, + UserId: userId, + DisplayName: cachedResolveDisplayName(userId, identityClient, cache), + Timestamp: ts, + FileName: record.Request.FileName, + FileExtension: filepath.Ext(record.Request.FileName), + HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "", + CompletionsCount: len(record.Response.Completions), + LeftContextLength: len(record.Request.LeftContext), + RightContextLength: len(record.Request.RightContext), + }, nil +} + +// normalizeUserId strips the "d-{directoryId}." prefix from Identity Center user IDs +// so that logging user IDs match the short UUID format used in user-report CSVs. +func normalizeUserId(userId string) string { + if idx := strings.LastIndex(userId, "."); idx != -1 && strings.HasPrefix(userId, "d-") { + return userId[idx+1:] + } + return userId +} + +var ExtractQDevLoggingDataMeta = plugin.SubTaskMeta{ + Name: "extractQDevLoggingData", + EntryPoint: ExtractQDevLoggingData, + EnabledByDefault: true, + Description: "Extract logging data from S3 JSON.gz files (chat and completion events)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta}, +} diff --git a/backend/plugins/q_dev/tasks/task_data.go b/backend/plugins/q_dev/tasks/task_data.go index 00c58f11e49..3fd3c65848d 100644 --- a/backend/plugins/q_dev/tasks/task_data.go +++ b/backend/plugins/q_dev/tasks/task_data.go @@ -29,12 +29,17 @@ type QDevOptions struct { ConnectionId uint64 `json:"connectionId"` S3Prefix string `json:"s3Prefix"` ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + BasePath string `json:"basePath"` + Year int `json:"year"` + Month *int `json:"month"` } type QDevTaskData struct { Options *QDevOptions S3Client *QDevS3Client - IdentityClient *QDevIdentityClient // New field for Identity Center client + IdentityClient *QDevIdentityClient + S3Prefixes []string } type QDevS3Client struct { diff --git a/backend/plugins/q_dev/tasks/task_data_test.go b/backend/plugins/q_dev/tasks/task_data_test.go index cb8f754378e..757f27428de 100644 --- a/backend/plugins/q_dev/tasks/task_data_test.go +++ b/backend/plugins/q_dev/tasks/task_data_test.go @@ -38,6 +38,7 @@ func TestQDevTaskData_WithIdentityClient(t *testing.T) { StoreId: "d-1234567890", Region: "us-west-2", }, + S3Prefixes: []string{"test-prefix/"}, } assert.NotNil(t, taskData.IdentityClient) @@ -45,6 +46,7 @@ func TestQDevTaskData_WithIdentityClient(t *testing.T) { assert.Equal(t, "us-west-2", taskData.IdentityClient.Region) assert.NotNil(t, taskData.S3Client) assert.NotNil(t, taskData.Options) + assert.Equal(t, []string{"test-prefix/"}, taskData.S3Prefixes) } func TestQDevTaskData_WithoutIdentityClient(t *testing.T) { @@ -68,9 +70,14 @@ func TestQDevTaskData_WithoutIdentityClient(t *testing.T) { } func TestQDevTaskData_AllFields(t *testing.T) { + month := 3 options := &QDevOptions{ ConnectionId: 123, S3Prefix: "data/q-dev/", + AccountId: "034362076319", + BasePath: "user-report", + Year: 2026, + Month: &month, } s3Client := &QDevS3Client{ @@ -87,6 +94,10 @@ func TestQDevTaskData_AllFields(t *testing.T) { Options: options, S3Client: s3Client, IdentityClient: identityClient, + S3Prefixes: []string{ + "user-report/AWSLogs/034362076319/KiroLogs/by_user_analytic/us-east-1/2026/03", + "user-report/AWSLogs/034362076319/KiroLogs/user_report/us-east-1/2026/03", + }, } // Verify all fields are properly set @@ -97,9 +108,14 @@ func TestQDevTaskData_AllFields(t *testing.T) { // Verify nested field access assert.Equal(t, uint64(123), taskData.Options.ConnectionId) assert.Equal(t, "data/q-dev/", taskData.Options.S3Prefix) + assert.Equal(t, "034362076319", taskData.Options.AccountId) + assert.Equal(t, "user-report", taskData.Options.BasePath) + assert.Equal(t, 2026, taskData.Options.Year) + assert.Equal(t, &month, taskData.Options.Month) assert.Equal(t, "my-data-bucket", taskData.S3Client.Bucket) assert.Equal(t, "d-9876543210", taskData.IdentityClient.StoreId) assert.Equal(t, "eu-west-1", taskData.IdentityClient.Region) + assert.Len(t, taskData.S3Prefixes, 2) } func TestQDevTaskData_EmptyStruct(t *testing.T) { diff --git a/backend/plugins/refdiff/tasks/refdiff_task_data.go b/backend/plugins/refdiff/tasks/refdiff_task_data.go index 782bb921419..c4bc074a234 100644 --- a/backend/plugins/refdiff/tasks/refdiff_task_data.go +++ b/backend/plugins/refdiff/tasks/refdiff_task_data.go @@ -121,7 +121,7 @@ func (rs RefsReverseSemver) Swap(i, j int) { func CalculateTagPattern(db dal.Dal, tagsPattern string, tagsLimit int, tagsOrder string) (Refs, errors.Error) { rs := Refs{} - // caculate Pattern part + // calculate Pattern part if tagsPattern == "" || tagsLimit <= 1 { return rs, nil } @@ -176,7 +176,7 @@ func CalculateCommitPairs(db dal.Dal, repoId string, pairs []models.RefPair, rs commitPairs = append(commitPairs, [4]string{rs[i-1].CommitSha, rs[i].CommitSha, rs[i-1].Name, rs[i].Name}) } - // caculate pairs part + // calculate pairs part // convert ref pairs into commit pairs ref2sha := func(refName string) (string, error) { ref := &code.Ref{} @@ -186,7 +186,7 @@ func CalculateCommitPairs(db dal.Dal, repoId string, pairs []models.RefPair, rs ref.Id = fmt.Sprintf("%s:%s", repoId, refName) err := db.First(ref) if err != nil && !db.IsErrorNotFound(err) { - return "", errors.NotFound.Wrap(err, fmt.Sprintf("faild to load Ref info for repoId:%s, refName:%s", repoId, refName)) + return "", errors.NotFound.Wrap(err, fmt.Sprintf("failed to load Ref info for repoId:%s, refName:%s", repoId, refName)) } return ref.CommitSha, nil } diff --git a/backend/plugins/rootly/api/blueprint_v200.go b/backend/plugins/rootly/api/blueprint_v200.go new file mode 100644 index 00000000000..9871beffdee --- /dev/null +++ b/backend/plugins/rootly/api/blueprint_v200.go @@ -0,0 +1,103 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/apache/incubator-devlake/plugins/rootly/tasks" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.Service, models.RootlyScopeConfig], + connection *models.RootlyConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := api.MakePipelinePlanTask( + "rootly", + subtaskMetas, + scopeConfig.Entities, + tasks.RootlyOptions{ + ConnectionId: connection.ID, + ServiceId: scope.Id, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.Service, models.RootlyScopeConfig], + connection *models.RootlyConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + + idgen := didgen.NewDomainIdGenerator(&models.Service{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.Id) + + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + + return scopes, nil +} diff --git a/backend/plugins/rootly/api/connection_api.go b/backend/plugins/rootly/api/connection_api.go new file mode 100644 index 00000000000..3fb36609ded --- /dev/null +++ b/backend/plugins/rootly/api/connection_api.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +func testConnection(ctx context.Context, connection models.RootlyConn) (*plugin.ApiResourceOutput, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + response, err := apiClient.Get("users/me", nil, nil) + if err != nil { + return nil, err + } + if response.StatusCode == http.StatusUnauthorized { + return nil, errors.HttpStatus(http.StatusBadRequest).New("StatusUnauthorized error while testing connection") + } + if response.StatusCode == http.StatusOK { + return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil + } + return &plugin.ApiResourceOutput{Body: nil, Status: response.StatusCode}, errors.HttpStatus(response.StatusCode).Wrap(err, "could not validate connection") +} + +// TestConnection test rootly connection +// @Summary test rootly connection +// @Description Test Rootly Connection +// @Tags plugins/rootly +// @Param body body models.RootlyConn true "json body" +// @Success 200 {object} shared.ApiBody "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.RootlyConn + err := api.Decode(input.Body, &connection, vld) + if err != nil { + return nil, err + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return testConnectionResult, nil +} + +// TestExistingConnection test rootly connection +// @Summary test rootly connection +// @Description Test Rootly Connection +// @Tags plugins/rootly +// @Param connectionId path int true "connection ID" +// @Success 200 {object} shared.ApiBody "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := api.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + testConnectionResult, testConnectionErr := testConnection(context.TODO(), connection.RootlyConn) + if testConnectionErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testConnectionErr) + } + return testConnectionResult, nil +} + +// @Summary create rootly connection +// @Description Create Rootly connection +// @Tags plugins/rootly +// @Param body body models.RootlyConnection true "json body" +// @Success 200 {object} models.RootlyConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// @Summary patch rootly connection +// @Description Patch Rootly connection +// @Tags plugins/rootly +// @Param body body models.RootlyConnection true "json body" +// @Success 200 {object} models.RootlyConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// @Summary delete rootly connection +// @Description Delete Rootly connection +// @Tags plugins/rootly +// @Success 200 {object} models.RootlyConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// @Summary list rootly connections +// @Description List Rootly connections +// @Tags plugins/rootly +// @Success 200 {object} models.RootlyConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// @Summary get rootly connection +// @Description Get Rootly connection +// @Tags plugins/rootly +// @Success 200 {object} models.RootlyConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/rootly/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/rootly/api/init.go b/backend/plugins/rootly/api/init.go new file mode 100644 index 00000000000..3c16101329b --- /dev/null +++ b/backend/plugins/rootly/api/init.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes + +var dsHelper *api.DsHelper[models.RootlyConnection, models.Service, models.RootlyScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.RootlyConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.RootlyConnection, models.Service, RootlyRemotePagination] + +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.RootlyConnection, models.Service] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + vld = validator.New() + basicRes = br + dsHelper = api.NewDataSourceHelper[ + models.RootlyConnection, models.Service, models.RootlyScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.RootlyConnection) models.RootlyConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.RootlyConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.RootlyConnection, models.Service, RootlyRemotePagination](raProxy, listRootlyRemoteScopes) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.RootlyConnection, models.Service](raProxy, searchRootlyRemoteScopes) +} diff --git a/backend/plugins/rootly/api/remote_api.go b/backend/plugins/rootly/api/remote_api.go new file mode 100644 index 00000000000..0bb81ce219d --- /dev/null +++ b/backend/plugins/rootly/api/remote_api.go @@ -0,0 +1,189 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/models/common" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +type RootlyRemotePagination struct { + Page int `json:"page"` + PerPage int `json:"per_page"` +} + +type ServiceResponse struct { + Data []struct { + Id string `json:"id"` + Type string `json:"type"` + Attributes struct { + Name string `json:"name"` + Slug string `json:"slug"` + Description *string `json:"description"` + HtmlUrl *string `json:"html_url"` + CreatedAt *time.Time `json:"created_at"` + } `json:"attributes"` + } `json:"data"` + Meta struct { + TotalCount int `json:"total_count"` + TotalPages int `json:"total_pages"` + CurrentPage int `json:"current_page"` + } `json:"meta"` +} + +func queryRootlyRemoteScopes( + apiClient plugin.ApiClient, + _ string, + page RootlyRemotePagination, + search string, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.Service], + nextPage *RootlyRemotePagination, + err errors.Error, +) { + if page.PerPage == 0 { + page.PerPage = 50 + } + if page.Page == 0 { + page.Page = 1 + } + query := url.Values{ + "page[number]": {strconv.Itoa(page.Page)}, + "page[size]": {strconv.Itoa(page.PerPage)}, + } + if search != "" { + query.Set("filter[search]", search) + } + var res *http.Response + res, err = apiClient.Get("services", query, nil) + if err != nil { + return + } + response := &ServiceResponse{} + err = api.UnmarshalResponse(res, response) + if err != nil { + return + } + for _, item := range response.Data { + htmlUrl := "" + if item.Attributes.HtmlUrl != nil { + htmlUrl = *item.Attributes.HtmlUrl + } + entry := dsmodels.DsRemoteApiScopeListEntry[models.Service]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: item.Id, + Name: item.Attributes.Name, + FullName: item.Attributes.Name, + Data: &models.Service{ + Url: htmlUrl, + Id: item.Id, + Name: item.Attributes.Name, + Scope: common.Scope{ + NoPKModel: common.NoPKModel{}, + }, + }, + } + if item.Attributes.CreatedAt != nil { + entry.Data.Scope.NoPKModel.CreatedAt = *item.Attributes.CreatedAt + } + children = append(children, entry) + } + + if page.Page < response.Meta.TotalPages { + nextPage = &RootlyRemotePagination{ + Page: page.Page + 1, + PerPage: page.PerPage, + } + } + + return +} + +func listRootlyRemoteScopes( + connection *models.RootlyConnection, + apiClient plugin.ApiClient, + groupId string, + page RootlyRemotePagination, +) ( + []dsmodels.DsRemoteApiScopeListEntry[models.Service], + *RootlyRemotePagination, + errors.Error, +) { + return queryRootlyRemoteScopes(apiClient, groupId, page, "") +} + +func searchRootlyRemoteScopes( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.Service], + err errors.Error, +) { + page := params.Page + if page == 0 { + page = 1 + } + children, _, err = queryRootlyRemoteScopes(apiClient, "", RootlyRemotePagination{ + Page: page, + PerPage: params.PageSize, + }, params.Search) + return +} + +// RemoteScopes list all available scopes (services) for this connection +// @Summary list all available scopes (services) for this connection +// @Description list all available scopes (services) for this connection +// @Tags plugins/rootly +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} RemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// SearchRemoteScopes use the Search API and only return project +// @Summary use the Search API and only return project +// @Description use the Search API and only return project +// @Tags plugins/rootly +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} SearchRemoteScopesOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} diff --git a/backend/plugins/rootly/api/scope_api.go b/backend/plugins/rootly/api/scope_api.go new file mode 100644 index 00000000000..ef8633423f6 --- /dev/null +++ b/backend/plugins/rootly/api/scope_api.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.Service] +type ScopeDetail api.ScopeDetail[models.Service, models.RootlyScopeConfig] + +// PutScopes create or update rootly service +// @Summary create or update rootly service +// @Description Create or update rootly service +// @Tags plugins/rootly +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scope body ScopeReq true "json" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to rootly service +// @Summary patch to rootly service +// @Description patch to rootly service +// @Tags plugins/rootly +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param scope body models.Service true "json" +// @Success 200 {object} models.Service +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get Rootly services +// @Summary get Rootly services +// @Description get Rootly services +// @Tags plugins/rootly +// @Param connectionId path int true "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one Rootly service +// @Summary get one Rootly service +// @Description get one Rootly service +// @Tags plugins/rootly +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param blueprints query bool false "also return blueprints using this scope as part of the payload" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/rootly +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/rootly/api/scope_state_api.go b/backend/plugins/rootly/api/scope_state_api.go new file mode 100644 index 00000000000..01be7dc951e --- /dev/null +++ b/backend/plugins/rootly/api/scope_state_api.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// GetScopeLatestSyncState get one rootly service's latest sync state +// @Summary get one rootly service's latest sync state +// @Description get one rootly service's latest sync state +// @Tags plugins/rootly +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/rootly/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/rootly/api/swagger.go b/backend/plugins/rootly/api/swagger.go new file mode 100644 index 00000000000..aa60b939865 --- /dev/null +++ b/backend/plugins/rootly/api/swagger.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/plugins/rootly/tasks" +) + +type RootlyTaskOptions tasks.RootlyOptions + +// @Summary rootly task options for pipelines +// @Description This is a dummy API to demonstrate the available task options for rootly pipelines +// @Tags plugins/rootly +// @Accept application/json +// @Param pipeline body RootlyTaskOptions true "json" +// @Router /pipelines/rootly/pipeline-task [post] +func _() {} diff --git a/backend/plugins/rootly/e2e/incident_test.go b/backend/plugins/rootly/e2e/incident_test.go new file mode 100644 index 00000000000..15cc1d324a3 --- /dev/null +++ b/backend/plugins/rootly/e2e/incident_test.go @@ -0,0 +1,126 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/rootly/impl" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/apache/incubator-devlake/plugins/rootly/tasks" + "github.com/stretchr/testify/require" +) + +func TestIncidentDataFlow(t *testing.T) { + var plugin impl.Rootly + dataflowTester := e2ehelper.NewDataFlowTester(t, "rootly", plugin) + options := tasks.RootlyOptions{ + ConnectionId: 1, + ServiceId: "svc_01", + ServiceName: "Payments", + } + taskData := &tasks.RootlyTaskData{ + Options: &options, + } + + // scope + dataflowTester.FlushTabler(&models.Service{}) + service := models.Service{ + Scope: common.Scope{ + ConnectionId: options.ConnectionId, + }, + Url: fmt.Sprintf("https://rootly.com/account/services/%s", options.ServiceId), + Id: options.ServiceId, + Name: options.ServiceName, + } + require.NoError(t, dataflowTester.Dal.CreateOrUpdate(&service)) + + // import raw data table + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_rootly_incidents.csv", + "_raw_rootly_incidents", + ) + + // verify extraction + dataflowTester.FlushTabler(&models.Incident{}) + dataflowTester.FlushTabler(&models.User{}) + dataflowTester.Subtask(tasks.ExtractIncidentsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + models.Service{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_rootly_services.csv", + IgnoreTypes: []any{common.Scope{}}, + }, + ) + dataflowTester.VerifyTableWithOptions( + models.Incident{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_rootly_incidents.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + dataflowTester.VerifyTableWithOptions( + models.User{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_rootly_users.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + + // verify conversion + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertServicesMeta, taskData) + dataflowTester.VerifyTableWithOptions( + ticket.Board{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIncidentsMeta, taskData) + dataflowTester.VerifyTableWithOptions( + ticket.Issue{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + IgnoreFields: []string{"original_project"}, + }, + ) + dataflowTester.VerifyTableWithOptions( + ticket.IssueAssignee{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) + dataflowTester.VerifyTableWithOptions( + ticket.BoardIssue{}, + e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []any{common.NoPKModel{}}, + }, + ) +} diff --git a/backend/plugins/rootly/e2e/raw_tables/_raw_rootly_incidents.csv b/backend/plugins/rootly/e2e/raw_tables/_raw_rootly_incidents.csv new file mode 100644 index 00000000000..cccd06ffc30 --- /dev/null +++ b/backend/plugins/rootly/e2e/raw_tables/_raw_rootly_incidents.csv @@ -0,0 +1,7 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_01"",""type"":""incidents"",""attributes"":{""sequential_id"":101,""title"":""Payment processor outage"",""summary"":""Payments are timing out"",""url"":""https://rootly.com/account/incidents/inc_01"",""status"":""triage"",""severity"":{""data"":{""id"":""sev-uuid-0"",""type"":""severities"",""attributes"":{""slug"":""sev0"",""name"":""SEV0"",""severity"":""critical""}}},""started_at"":""2026-05-01T10:00:00Z"",""updated_at"":""2026-05-01T10:05:00Z"",""user"":{""data"":{""id"":""u1"",""type"":""users"",""attributes"":{""email"":""alice@example.com"",""full_name"":""Alice""}}}},""relationships"":{""services"":{""data"":[{""id"":""svc_01"",""type"":""services""}]}}}",,null,2026-05-01T10:05:00.000+00:00 +2,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_02"",""type"":""incidents"",""attributes"":{""sequential_id"":102,""title"":""Latency spike"",""summary"":""p99 latency above SLO"",""url"":""https://rootly.com/account/incidents/inc_02"",""status"":""mitigated"",""severity"":{""data"":{""id"":""sev-uuid-1"",""type"":""severities"",""attributes"":{""slug"":""sev1"",""name"":""SEV1"",""severity"":""high""}}},""started_at"":""2026-05-02T09:00:00Z"",""mitigated_at"":""2026-05-02T09:45:00Z"",""updated_at"":""2026-05-02T09:45:00Z"",""user"":{""data"":{""id"":""u1"",""type"":""users"",""attributes"":{""email"":""alice@example.com"",""full_name"":""Alice""}}},""started_by"":{""data"":{""id"":""u2"",""type"":""users"",""attributes"":{""email"":""bob@example.com"",""full_name"":""Bob""}}},""mitigated_by"":{""data"":{""id"":""u2"",""type"":""users"",""attributes"":{""email"":""bob@example.com"",""full_name"":""Bob""}}}},""relationships"":{""services"":{""data"":[{""id"":""svc_01"",""type"":""services""}]}}}",,null,2026-05-02T09:45:00.000+00:00 +3,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_03"",""type"":""incidents"",""attributes"":{""sequential_id"":103,""title"":""Queue backlog"",""summary"":""Worker queue backed up"",""url"":""https://rootly.com/account/incidents/inc_03"",""status"":""resolved"",""severity"":{""data"":{""id"":""sev-uuid-2"",""type"":""severities"",""attributes"":{""slug"":""sev2"",""name"":""SEV2"",""severity"":""medium""}}},""started_at"":""2026-05-03T12:00:00Z"",""resolved_at"":""2026-05-03T13:30:00Z"",""updated_at"":""2026-05-03T13:30:00Z"",""user"":{""data"":{""id"":""u1"",""type"":""users"",""attributes"":{""email"":""alice@example.com"",""full_name"":""Alice""}}},""resolved_by"":{""data"":{""id"":""u1"",""type"":""users"",""attributes"":{""email"":""alice@example.com"",""full_name"":""Alice""}}}},""relationships"":{""services"":{""data"":[{""id"":""svc_01"",""type"":""services""}]}}}",,null,2026-05-03T13:30:00.000+00:00 +4,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_04"",""type"":""incidents"",""attributes"":{""sequential_id"":201,""title"":""Wrong-service incident"",""summary"":""This incident belongs to svc_02"",""url"":""https://rootly.com/account/incidents/inc_04"",""status"":""closed"",""severity"":{""data"":{""id"":""sev-uuid-3"",""type"":""severities"",""attributes"":{""slug"":""sev3"",""name"":""SEV3"",""severity"":""low""}}},""started_at"":""2026-05-04T08:00:00Z"",""resolved_at"":""2026-05-04T08:30:00Z"",""updated_at"":""2026-05-04T08:30:00Z"",""user"":{""data"":{""id"":""u2"",""type"":""users"",""attributes"":{""email"":""bob@example.com"",""full_name"":""Bob""}}}},""relationships"":{""services"":{""data"":[{""id"":""svc_02"",""type"":""services""}]}}}",,null,2026-05-04T08:30:00.000+00:00 +5,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_05"",""type"":""incidents"",""attributes"":{""sequential_id"":104,""title"":""False alarm"",""summary"":""Cancelled after triage"",""url"":""https://rootly.com/account/incidents/inc_05"",""status"":""cancelled"",""severity"":{""data"":{""id"":""sev-uuid-4"",""type"":""severities"",""attributes"":{""slug"":""sev4"",""name"":""SEV4"",""severity"":""low""}}},""started_at"":""2026-05-05T14:00:00Z"",""updated_at"":""2026-05-05T14:05:00Z""},""relationships"":{""services"":{""data"":[{""id"":""svc_01"",""type"":""services""}]}}}",,null,2026-05-05T14:05:00.000+00:00 +6,"{""ConnectionId"":1,""ScopeId"":""svc_01""}","{""id"":""inc_06"",""type"":""incidents"",""attributes"":{""sequential_id"":105,""title"":""Odd state"",""summary"":""Status from future Rootly version"",""url"":""https://rootly.com/account/incidents/inc_06"",""status"":""investigating"",""severity"":{""data"":{""id"":""sev-uuid-x"",""type"":""severities"",""attributes"":{""slug"":""blocker"",""name"":""Blocker"",""severity"":""critical""}}},""started_at"":""2026-05-06T11:00:00Z"",""updated_at"":""2026-05-06T11:10:00Z"",""user"":{""data"":{""id"":""u3"",""type"":""users"",""attributes"":{""email"":""carol@example.com"",""full_name"":""Carol""}}}},""relationships"":{""services"":{""data"":[{""id"":""svc_01"",""type"":""services""}]}}}",,null,2026-05-06T11:10:00.000+00:00 diff --git a/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_incidents.csv b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_incidents.csv new file mode 100644 index 00000000000..7a180306867 --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_incidents.csv @@ -0,0 +1,6 @@ +connection_id,id,number,service_id,url,title,summary,status,severity,started_date,acknowledged_date,mitigated_date,resolved_date,updated_date,creator_user_id,started_by_user_id,mitigated_by_user_id,resolved_by_user_id,closed_by_user_id +1,inc_01,101,svc_01,https://rootly.com/account/incidents/inc_01,Payment processor outage,Payments are timing out,triage,sev0,2026-05-01T10:00:00.000+00:00,,,,2026-05-01T10:05:00.000+00:00,u1,,,, +1,inc_02,102,svc_01,https://rootly.com/account/incidents/inc_02,Latency spike,p99 latency above SLO,mitigated,sev1,2026-05-02T09:00:00.000+00:00,,2026-05-02T09:45:00.000+00:00,,2026-05-02T09:45:00.000+00:00,u1,u2,u2,, +1,inc_03,103,svc_01,https://rootly.com/account/incidents/inc_03,Queue backlog,Worker queue backed up,resolved,sev2,2026-05-03T12:00:00.000+00:00,,,2026-05-03T13:30:00.000+00:00,2026-05-03T13:30:00.000+00:00,u1,,,u1, +1,inc_05,104,svc_01,https://rootly.com/account/incidents/inc_05,False alarm,Cancelled after triage,cancelled,sev4,2026-05-05T14:00:00.000+00:00,,,,2026-05-05T14:05:00.000+00:00,,,,, +1,inc_06,105,svc_01,https://rootly.com/account/incidents/inc_06,Odd state,Status from future Rootly version,investigating,blocker,2026-05-06T11:00:00.000+00:00,,,,2026-05-06T11:10:00.000+00:00,u3,,,, diff --git a/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_services.csv b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_services.csv new file mode 100644 index 00000000000..e5029d1d698 --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_services.csv @@ -0,0 +1,2 @@ +connection_id,id,url,name +1,svc_01,https://rootly.com/account/services/svc_01,Payments diff --git a/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_users.csv b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_users.csv new file mode 100644 index 00000000000..759cc859347 --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/_tool_rootly_users.csv @@ -0,0 +1,4 @@ +connection_id,id,email,name,url +1,u1,alice@example.com,Alice, +1,u2,bob@example.com,Bob, +1,u3,carol@example.com,Carol, diff --git a/backend/plugins/rootly/e2e/snapshot_tables/board_issues.csv b/backend/plugins/rootly/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..654e122979e --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +rootly:Service:1:svc_01,rootly:Incident:1:inc_01 +rootly:Service:1:svc_01,rootly:Incident:1:inc_02 +rootly:Service:1:svc_01,rootly:Incident:1:inc_03 +rootly:Service:1:svc_01,rootly:Incident:1:inc_05 +rootly:Service:1:svc_01,rootly:Incident:1:inc_06 diff --git a/backend/plugins/rootly/e2e/snapshot_tables/boards.csv b/backend/plugins/rootly/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..20bc5a3e054 --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +rootly:Service:1:svc_01,Payments,,https://rootly.com/account/services/svc_01,, diff --git a/backend/plugins/rootly/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/rootly/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..6a4ecdb1f60 --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,6 @@ +issue_id,assignee_id,assignee_name +rootly:Incident:1:inc_01,rootly:User:1:u1,Alice +rootly:Incident:1:inc_02,rootly:User:1:u1,Alice +rootly:Incident:1:inc_02,rootly:User:1:u2,Bob +rootly:Incident:1:inc_03,rootly:User:1:u1,Alice +rootly:Incident:1:inc_06,rootly:User:1:u3,Carol diff --git a/backend/plugins/rootly/e2e/snapshot_tables/issues.csv b/backend/plugins/rootly/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..43c50aa8e3a --- /dev/null +++ b/backend/plugins/rootly/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,is_subtask,due_date,fix_versions +rootly:Incident:1:inc_01,https://rootly.com/account/incidents/inc_01,,101,Payment processor outage,Payments are timing out,,INCIDENT,,TODO,triage,,,2026-05-01T10:00:00.000+00:00,2026-05-01T10:05:00.000+00:00,,,,,rootly:User:1:u1,Alice,rootly:User:1:u1,Alice,,CRITICAL,sev0,,,0,, +rootly:Incident:1:inc_02,https://rootly.com/account/incidents/inc_02,,102,Latency spike,p99 latency above SLO,,INCIDENT,,IN_PROGRESS,mitigated,,,2026-05-02T09:00:00.000+00:00,2026-05-02T09:45:00.000+00:00,,,,,rootly:User:1:u1,Alice,rootly:User:1:u1,Alice,,HIGH,sev1,,,0,, +rootly:Incident:1:inc_03,https://rootly.com/account/incidents/inc_03,,103,Queue backlog,Worker queue backed up,,INCIDENT,,DONE,resolved,,2026-05-03T13:30:00.000+00:00,2026-05-03T12:00:00.000+00:00,2026-05-03T13:30:00.000+00:00,90,,,,rootly:User:1:u1,Alice,rootly:User:1:u1,Alice,,MEDIUM,sev2,,,0,, +rootly:Incident:1:inc_05,https://rootly.com/account/incidents/inc_05,,104,False alarm,Cancelled after triage,,INCIDENT,,DONE,cancelled,,,2026-05-05T14:00:00.000+00:00,2026-05-05T14:05:00.000+00:00,,,,,,,,,,LOW,sev4,,,0,, +rootly:Incident:1:inc_06,https://rootly.com/account/incidents/inc_06,,105,Odd state,Status from future Rootly version,,INCIDENT,,IN_PROGRESS,investigating,,,2026-05-06T11:00:00.000+00:00,2026-05-06T11:10:00.000+00:00,,,,,rootly:User:1:u3,Carol,rootly:User:1:u3,Carol,,blocker,blocker,,,0,, diff --git a/backend/plugins/rootly/impl/impl.go b/backend/plugins/rootly/impl/impl.go new file mode 100644 index 00000000000..296bd6e6594 --- /dev/null +++ b/backend/plugins/rootly/impl/impl.go @@ -0,0 +1,190 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/apache/incubator-devlake/plugins/rootly/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/rootly/tasks" +) + +// make sure interface is implemented + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Rootly)(nil) + +type Rootly struct{} + +func (p Rootly) Description() string { + return "collect Rootly incident data" +} + +func (p Rootly) Name() string { + return "rootly" +} + +func (p Rootly) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Rootly) Connection() dal.Tabler { + return &models.RootlyConnection{} +} + +func (p Rootly) Scope() plugin.ToolLayerScope { + return &models.Service{} +} + +func (p Rootly) ScopeConfig() dal.Tabler { + return &models.RootlyScopeConfig{} +} + +func (p Rootly) SubTaskMetas() []plugin.SubTaskMeta { + // Convert services before incidents so the domain Board row exists + // before the BoardIssue rows that reference it; the opposite order + // (PagerDuty / Opsgenie convention) works too because BoardIssue + // has no FK enforcement, but ours is explicit about the dependency. + return []plugin.SubTaskMeta{ + tasks.CollectServicesMeta, + tasks.ExtractServicesMeta, + tasks.CollectIncidentsMeta, + tasks.ExtractIncidentsMeta, + tasks.ConvertServicesMeta, + tasks.ConvertIncidentsMeta, + } +} + +func (p Rootly) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.Service{}, + &models.Incident{}, + &models.User{}, + &models.RootlyConnection{}, + &models.RootlyScopeConfig{}, + } +} + +func (p Rootly) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + connection := &models.RootlyConnection{} + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Rootly connection by the given connection ID") + } + + client, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + asyncClient, err := helper.CreateAsyncApiClient(taskCtx, client, nil) + if err != nil { + return nil, err + } + return &tasks.RootlyTaskData{ + Options: op, + Client: asyncClient, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Rootly) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/rootly" +} + +func (p Rootly) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Rootly) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + } +} + +func (p Rootly) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Rootly) Close(taskCtx plugin.TaskContext) errors.Error { + _, ok := taskCtx.GetData().(*tasks.RootlyTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + return nil +} diff --git a/backend/plugins/rootly/models/connection.go b/backend/plugins/rootly/models/connection.go new file mode 100644 index 00000000000..898d1542198 --- /dev/null +++ b/backend/plugins/rootly/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type RootlyAccessToken helper.AccessToken + +func (at *RootlyAccessToken) SetupAuthentication(request *http.Request) errors.Error { + request.Header.Set("Authorization", fmt.Sprintf("Bearer %s", at.Token)) + return nil +} + +type RootlyConn struct { + helper.RestConnection `mapstructure:",squash"` + RootlyAccessToken `mapstructure:",squash"` +} + +func (connection RootlyConn) Sanitize() RootlyConn { + connection.Token = utils.SanitizeString(connection.Token) + return connection +} + +type RootlyConnection struct { + helper.BaseConnection `mapstructure:",squash"` + RootlyConn `mapstructure:",squash"` +} + +// MergeFromRequest preserves the existing token when an incoming PATCH +// body omits it or echoes the sanitized form. The config-UI sends the +// sanitized token back on every PATCH to avoid round-tripping the +// secret; this guard is what makes that pattern safe. +func (connection *RootlyConnection) MergeFromRequest(target *RootlyConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (RootlyConnection) TableName() string { + return "_tool_rootly_connections" +} + +func (connection RootlyConnection) Sanitize() RootlyConnection { + connection.Token = utils.SanitizeString(connection.Token) + return connection +} diff --git a/backend/plugins/rootly/models/incident.go b/backend/plugins/rootly/models/incident.go new file mode 100644 index 00000000000..804ed645798 --- /dev/null +++ b/backend/plugins/rootly/models/incident.go @@ -0,0 +1,62 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type Incident struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Number int + ServiceId string `gorm:"index"` + Url string + Title string + Summary string + Status string + Severity string + StartedDate time.Time + AcknowledgedDate *time.Time + MitigatedDate *time.Time + ResolvedDate *time.Time + UpdatedDate time.Time + CreatorUserId string + StartedByUserId string + MitigatedByUserId string + ResolvedByUserId string + ClosedByUserId string +} + +func (Incident) TableName() string { return "_tool_rootly_incidents" } + +// RoleUserIds returns the five role-bearing user ids on the incident +// in lifecycle order (creator, started_by, mitigated_by, resolved_by, +// closed_by). Empty strings are included; callers filter or dedup. +func (i *Incident) RoleUserIds() []string { + return []string{ + i.CreatorUserId, + i.StartedByUserId, + i.MitigatedByUserId, + i.ResolvedByUserId, + i.ClosedByUserId, + } +} diff --git a/backend/plugins/rootly/models/migrationscripts/20260512_add_init_tables.go b/backend/plugins/rootly/models/migrationscripts/20260512_add_init_tables.go new file mode 100644 index 00000000000..ec4ee5fcad6 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/20260512_add_init_tables.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/rootly/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(baseRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(baseRes, + &archived.Connection{}, + &archived.Service{}, + &archived.Incident{}, + &archived.User{}, + &archived.ScopeConfig{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260512000001 +} + +func (*addInitTables) Name() string { + return "Rootly init schemas" +} diff --git a/backend/plugins/rootly/models/migrationscripts/archived/connection.go b/backend/plugins/rootly/models/migrationscripts/archived/connection.go new file mode 100644 index 00000000000..5bfd1ad47a5 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/archived/connection.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type Connection struct { + archived.Model + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name" validate:"required"` + Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimitPerHour"` + Token string `mapstructure:"token" env:"ROOTLY_AUTH" validate:"required" encrypt:"yes"` +} + +func (Connection) TableName() string { + return "_tool_rootly_connections" +} diff --git a/backend/plugins/rootly/models/migrationscripts/archived/incident.go b/backend/plugins/rootly/models/migrationscripts/archived/incident.go new file mode 100644 index 00000000000..b787b8c81ff --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/archived/incident.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type Incident struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Number int + ServiceId string `gorm:"index"` + Url string + Title string + Summary string + Status string + Severity string + StartedDate time.Time + AcknowledgedDate *time.Time + MitigatedDate *time.Time + ResolvedDate *time.Time + UpdatedDate time.Time + CreatorUserId string + StartedByUserId string + MitigatedByUserId string + ResolvedByUserId string + ClosedByUserId string +} + +func (Incident) TableName() string { + return "_tool_rootly_incidents" +} diff --git a/backend/plugins/rootly/models/migrationscripts/archived/scope_config.go b/backend/plugins/rootly/models/migrationscripts/archived/scope_config.go new file mode 100644 index 00000000000..4438c38f480 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/archived/scope_config.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +// ConnectionId and Name come from `common.ScopeConfig` on the live +// model; the archived `archived.ScopeConfig` base only carries +// Model + Entities, so declare them explicitly. +type ScopeConfig struct { + archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + ConnectionId uint64 `json:"connectionId" gorm:"index" validate:"required" mapstructure:"connectionId,omitempty"` + Name string `mapstructure:"name" json:"name" gorm:"type:varchar(255);uniqueIndex" validate:"required"` +} + +func (ScopeConfig) TableName() string { + return "_tool_rootly_scope_configs" +} diff --git a/backend/plugins/rootly/models/migrationscripts/archived/service.go b/backend/plugins/rootly/models/migrationscripts/archived/service.go new file mode 100644 index 00000000000..821b9ea2b01 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/archived/service.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +// ScopeConfigId mirrors the column that live `models.Service` gets via +// embedded `common.Scope`; the archived `NoPKModel` doesn't include it. +type Service struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Url string + Name string +} + +func (Service) TableName() string { + return "_tool_rootly_services" +} diff --git a/backend/plugins/rootly/models/migrationscripts/archived/user.go b/backend/plugins/rootly/models/migrationscripts/archived/user.go new file mode 100644 index 00000000000..d16851d3249 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/archived/user.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + +type User struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Email string + Name string +} + +func (User) TableName() string { + return "_tool_rootly_users" +} diff --git a/backend/plugins/rootly/models/migrationscripts/register.go b/backend/plugins/rootly/models/migrationscripts/register.go new file mode 100644 index 00000000000..d333899d309 --- /dev/null +++ b/backend/plugins/rootly/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All returns all the migration scripts for the rootly plugin +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/rootly/models/raw/incident.go b/backend/plugins/rootly/models/raw/incident.go new file mode 100644 index 00000000000..541fb477e5c --- /dev/null +++ b/backend/plugins/rootly/models/raw/incident.go @@ -0,0 +1,87 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package raw + +import ( + "time" +) + +type Incident struct { + Id string `json:"id"` + Type string `json:"type"` + Attributes IncidentAttributes `json:"attributes"` + Relationships IncidentRelationships `json:"relationships"` +} + +type IncidentRelationships struct { + Services struct { + Data []ServiceRef `json:"data"` + } `json:"services"` +} + +type ServiceRef struct { + Id string `json:"id"` + Type string `json:"type"` +} + +type IncidentAttributes struct { + SequentialId *int `json:"sequential_id"` + Title string `json:"title"` + Summary *string `json:"summary"` + Url *string `json:"url"` + Status string `json:"status"` + StartedAt time.Time `json:"started_at"` + AcknowledgedAt *time.Time `json:"acknowledged_at"` + MitigatedAt *time.Time `json:"mitigated_at"` + ResolvedAt *time.Time `json:"resolved_at"` + UpdatedAt time.Time `json:"updated_at"` + + Severity *SeverityEnvelope `json:"severity"` + + User *UserEnvelope `json:"user"` + StartedBy *UserEnvelope `json:"started_by"` + MitigatedBy *UserEnvelope `json:"mitigated_by"` + ResolvedBy *UserEnvelope `json:"resolved_by"` + ClosedBy *UserEnvelope `json:"closed_by"` +} + +type SeverityEnvelope struct { + Data struct { + Id string `json:"id"` + Type string `json:"type"` + Attributes SeverityAttributes `json:"attributes"` + } `json:"data"` +} + +type SeverityAttributes struct { + Slug string `json:"slug"` +} + +type UserEnvelope struct { + Data struct { + Id string `json:"id"` + Type string `json:"type"` + Attributes UserAttributes `json:"attributes"` + } `json:"data"` +} + +type UserAttributes struct { + Name string `json:"name"` + FullName string `json:"full_name"` + Email string `json:"email"` +} diff --git a/backend/plugins/rootly/models/raw/service.go b/backend/plugins/rootly/models/raw/service.go new file mode 100644 index 00000000000..d9971796274 --- /dev/null +++ b/backend/plugins/rootly/models/raw/service.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package raw + +import "time" + +type Service struct { + Id string `json:"id"` + Type string `json:"type"` + Attributes ServiceAttributes `json:"attributes"` +} + +type ServiceAttributes struct { + Name string `json:"name"` + Slug *string `json:"slug"` + Description *string `json:"description"` + HtmlUrl *string `json:"html_url"` + CreatedAt *time.Time `json:"created_at"` + UpdatedAt *time.Time `json:"updated_at"` +} diff --git a/backend/plugins/rootly/models/scope_config.go b/backend/plugins/rootly/models/scope_config.go new file mode 100644 index 00000000000..e2926554147 --- /dev/null +++ b/backend/plugins/rootly/models/scope_config.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type RootlyScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` +} + +func (RootlyScopeConfig) TableName() string { + return "_tool_rootly_scope_configs" +} diff --git a/backend/plugins/rootly/models/service.go b/backend/plugins/rootly/models/service.go new file mode 100644 index 00000000000..317cfe9f5a2 --- /dev/null +++ b/backend/plugins/rootly/models/service.go @@ -0,0 +1,60 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +type RootlyParams struct { + ConnectionId uint64 + ScopeId string +} + +type Service struct { + common.Scope `mapstructure:",squash"` + Id string `json:"id" mapstructure:"id" gorm:"primaryKey;autoIncrement:false" ` + Url string `json:"url" mapstructure:"url"` + Name string `json:"name" mapstructure:"name"` +} + +func (s Service) ScopeId() string { + return s.Id +} + +func (s Service) ScopeName() string { + return s.Name +} + +func (s Service) ScopeFullName() string { + return s.Name +} + +func (s Service) ScopeParams() interface{} { + return &RootlyParams{ + ConnectionId: s.ConnectionId, + ScopeId: s.Id, + } +} + +func (s Service) TableName() string { + return "_tool_rootly_services" +} + +var _ plugin.ToolLayerScope = (*Service)(nil) diff --git a/backend/plugins/rootly/models/user.go b/backend/plugins/rootly/models/user.go new file mode 100644 index 00000000000..4bf25434501 --- /dev/null +++ b/backend/plugins/rootly/models/user.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/models/common" + +type User struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;autoIncrement:false"` + Email string + Name string +} + +func (User) TableName() string { return "_tool_rootly_users" } diff --git a/backend/plugins/rootly/rootly.go b/backend/plugins/rootly/rootly.go new file mode 100644 index 00000000000..c1bb7d5001e --- /dev/null +++ b/backend/plugins/rootly/rootly.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/rootly/impl" + "github.com/spf13/cobra" +) + +// PluginEntry Export a variable named PluginEntry for Framework to search and load +var PluginEntry impl.Rootly //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "rootly"} + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{}, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/rootly/tasks/incidents_collector.go b/backend/plugins/rootly/tasks/incidents_collector.go new file mode 100644 index 00000000000..652801217aa --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_collector.go @@ -0,0 +1,140 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_INCIDENTS_TABLE = "rootly_incidents" + +var _ plugin.SubTaskEntryPoint = CollectIncidents + +type collectedIncidents struct { + Data []json.RawMessage `json:"data"` + Meta *collectedListMeta `json:"meta"` + Links *collectedListLinks `json:"links"` +} + +type collectedListMeta struct { + CurrentPage *int `json:"current_page"` + TotalPages *int `json:"total_pages"` +} + +type collectedListLinks struct { + Next *string `json:"next"` +} + +var CollectIncidentsMeta = plugin.SubTaskMeta{ + Name: "collectIncidents", + EntryPoint: CollectIncidents, + EnabledByDefault: true, + Description: "Collect Rootly incidents", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RAW_INCIDENTS_TABLE}, +} + +func CollectIncidents(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*RootlyTaskData) + args := api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + } + // Pagination state captured during ResponseParser and consulted in + // GetNextPageCustomData. Required because prevPageResponse.Body is + // a single-read stream and is already drained by the time the + // next-page hook fires. + var lastPage *collectedListMeta + var lastLinksNext *string + + collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{ + RawDataSubTaskArgs: args, + ApiClient: data.Client, + CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{ + PageSize: 100, + GetNextPageCustomData: func(prevReqData *api.RequestData, prevPageResponse *http.Response) (interface{}, errors.Error) { + // Safety cap against an upstream that returns full pages forever + // without populating either meta.total_pages or links.next. + const maxPages = 10000 + if prevReqData.Pager.Page >= maxPages { + return nil, api.ErrFinishCollect + } + if lastLinksNext != nil && *lastLinksNext != "" { + return nil, nil + } + if lastPage != nil && lastPage.CurrentPage != nil && lastPage.TotalPages != nil { + if *lastPage.CurrentPage >= *lastPage.TotalPages { + return nil, api.ErrFinishCollect + } + } + return nil, nil + }, + FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ + UrlTemplate: "incidents", + Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { + return buildIncidentsQuery(data.Options.ServiceId, reqData.Pager.Size, reqData.Pager.Page, createdAfter), nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + rawResult := collectedIncidents{} + if err := api.UnmarshalResponse(res, &rawResult); err != nil { + return nil, err + } + lastPage = rawResult.Meta + if rawResult.Links != nil { + lastLinksNext = rawResult.Links.Next + } else { + lastLinksNext = nil + } + return rawResult.Data, nil + }, + }, + }, + }) + if err != nil { + return err + } + return collector.Execute() +} + +// buildIncidentsQuery is the pure-function core of the Query closure +// above so a regression in the filter parameter name (we shipped with +// `filter[services]` once and got 0 results back; the correct param is +// `filter[service_ids]`) is caught by a unit test. +func buildIncidentsQuery(serviceId string, pageSize, pageNumber int, createdAfter *time.Time) url.Values { + query := url.Values{} + if serviceId != "" { + query.Set("filter[service_ids]", serviceId) + } + query.Set("page[size]", fmt.Sprintf("%d", pageSize)) + // Rootly's JSON:API pagination is 1-based. + query.Set("page[number]", fmt.Sprintf("%d", pageNumber)) + query.Set("sort", "-updated_at") + if createdAfter != nil { + query.Set("filter[updated_at][gt]", createdAfter.UTC().Format(time.RFC3339)) + } + return query +} diff --git a/backend/plugins/rootly/tasks/incidents_collector_test.go b/backend/plugins/rootly/tasks/incidents_collector_test.go new file mode 100644 index 00000000000..3456749a409 --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_collector_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBuildIncidentsQuery_FirstPageNoSince(t *testing.T) { + q := buildIncidentsQuery("svc_42", 100, 1, nil) + assert.Equal(t, "svc_42", q.Get("filter[service_ids]")) + assert.Equal(t, "100", q.Get("page[size]")) + assert.Equal(t, "1", q.Get("page[number]")) + assert.Equal(t, "-updated_at", q.Get("sort")) + assert.Equal(t, "", q.Get("filter[updated_at][gt]")) + assert.Equal(t, "", q.Get("filter[services]"), "regression guard: must be filter[service_ids], not filter[services]") +} + +func TestBuildIncidentsQuery_NoServiceFilter(t *testing.T) { + q := buildIncidentsQuery("", 100, 1, nil) + assert.Equal(t, "", q.Get("filter[service_ids]"), "empty serviceId must omit the service filter entirely") + assert.Equal(t, "100", q.Get("page[size]")) + assert.Equal(t, "1", q.Get("page[number]")) +} + +func TestBuildIncidentsQuery_SubsequentPage(t *testing.T) { + q := buildIncidentsQuery("svc_42", 100, 3, nil) + assert.Equal(t, "3", q.Get("page[number]")) +} + +func TestBuildIncidentsQuery_WithSince(t *testing.T) { + since := time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC) + q := buildIncidentsQuery("svc_42", 100, 1, &since) + assert.Equal(t, "2026-05-01T12:00:00Z", q.Get("filter[updated_at][gt]")) +} diff --git a/backend/plugins/rootly/tasks/incidents_converter.go b/backend/plugins/rootly/tasks/incidents_converter.go new file mode 100644 index 00000000000..daba86f505f --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_converter.go @@ -0,0 +1,212 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "strings" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +var _ plugin.SubTaskEntryPoint = ConvertIncidents + +var ConvertIncidentsMeta = plugin.SubTaskMeta{ + Name: "convertIncidents", + EntryPoint: ConvertIncidents, + EnabledByDefault: true, + Description: "Convert Rootly incidents into domain-layer ticket issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertIncidents(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*RootlyTaskData) + logger := taskCtx.GetLogger() + + var userRows []models.User + if err := db.All( + &userRows, + dal.Where("connection_id = ?", data.Options.ConnectionId), + ); err != nil { + return err + } + userNames := make(map[string]string, len(userRows)) + for _, u := range userRows { + userNames[u.Id] = u.Name + } + + cursor, err := db.Cursor( + dal.From(&models.Incident{}), + dal.Where("connection_id = ? AND service_id = ?", data.Options.ConnectionId, data.Options.ServiceId), + ) + if err != nil { + return err + } + defer cursor.Close() + + idGen := didgen.NewDomainIdGenerator(&models.Incident{}) + serviceIdGen := didgen.NewDomainIdGenerator(&models.Service{}) + userIdGen := didgen.NewDomainIdGenerator(&models.User{}) + boardId := serviceIdGen.Generate(data.Options.ConnectionId, data.Options.ServiceId) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.Incident{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + incident := inputRow.(*models.Incident) + + status, known := mapStatus(incident.Status) + if !known { + logger.Warn(nil, "unknown rootly incident status: %s", incident.Status) + } + + leadTime, resolutionDate := computeLeadTime(incident.StartedDate, incident.ResolvedDate, incident.UpdatedDate, incident.Status) + + domainIssueId := idGen.Generate(data.Options.ConnectionId, incident.Id) + + var creatorDomainId, creatorName string + if incident.CreatorUserId != "" { + creatorDomainId = userIdGen.Generate(data.Options.ConnectionId, incident.CreatorUserId) + creatorName = userNames[incident.CreatorUserId] + } + + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: domainIssueId, + }, + Url: incident.Url, + IssueKey: issueKeyFor(incident), + Title: incident.Title, + Description: incident.Summary, + Type: ticket.INCIDENT, + Status: status, + OriginalStatus: incident.Status, + ResolutionDate: resolutionDate, + CreatedDate: &incident.StartedDate, + UpdatedDate: &incident.UpdatedDate, + LeadTimeMinutes: leadTime, + Priority: mapSeverityToPriority(incident.Severity), + Severity: incident.Severity, + CreatorId: creatorDomainId, + CreatorName: creatorName, + AssigneeId: creatorDomainId, + AssigneeName: creatorName, + } + + results := []interface{}{domainIssue} + + seenAssignees := map[string]bool{} + for _, toolUserId := range incident.RoleUserIds() { + if toolUserId == "" || seenAssignees[toolUserId] { + continue + } + seenAssignees[toolUserId] = true + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssueId, + AssigneeId: userIdGen.Generate(data.Options.ConnectionId, toolUserId), + AssigneeName: userNames[toolUserId], + }) + } + + results = append(results, &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssueId, + }) + + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// Unknown statuses fall through to IN_PROGRESS rather than panicking +// (PagerDuty panics). Rootly's status enum is more volatile, so a new +// value from upstream shouldn't crash a production pipeline. +func mapStatus(status string) (mapped string, known bool) { + switch status { + case "triage", "started": + return ticket.TODO, true + case "mitigated": + return ticket.IN_PROGRESS, true + case "resolved", "closed", "cancelled", "completed": + return ticket.DONE, true + default: + return ticket.IN_PROGRESS, false + } +} + +func mapSeverityToPriority(severity string) string { + switch strings.ToLower(severity) { + case "sev0": + return "CRITICAL" + case "sev1": + return "HIGH" + case "sev2": + return "MEDIUM" + case "sev3", "sev4": + return "LOW" + default: + return severity + } +} + +func computeLeadTime(started time.Time, resolved *time.Time, updated time.Time, status string) (*uint, *time.Time) { + // For "completed" incidents Rootly may not populate resolved_date; fall + // back to updated_date which reflects when the incident was last actioned. + effective := resolved + if effective == nil && status == "completed" { + effective = &updated + } + if effective == nil { + return nil, nil + } + // Clock skew / backfill can place resolved before started. A naive + // uint() cast on a negative duration wraps to huge garbage and + // silently corrupts MTTR; treat as unresolved instead. + if effective.Before(started) { + return nil, nil + } + minutes := uint(effective.Sub(started).Minutes()) + resolutionDate := *effective + return &minutes, &resolutionDate +} + +func issueKeyFor(incident *models.Incident) string { + if incident.Number > 0 { + return fmt.Sprintf("%d", incident.Number) + } + return incident.Id +} diff --git a/backend/plugins/rootly/tasks/incidents_converter_test.go b/backend/plugins/rootly/tasks/incidents_converter_test.go new file mode 100644 index 00000000000..422f719eb6f --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_converter_test.go @@ -0,0 +1,223 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +func TestMapStatus(t *testing.T) { + cases := []struct { + in string + expectMapped string + expectedKnown bool + }{ + {"triage", ticket.TODO, true}, + {"started", ticket.TODO, true}, + {"mitigated", ticket.IN_PROGRESS, true}, + {"resolved", ticket.DONE, true}, + {"closed", ticket.DONE, true}, + {"cancelled", ticket.DONE, true}, + {"wat", ticket.IN_PROGRESS, false}, + {"", ticket.IN_PROGRESS, false}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + mapped, known := mapStatus(c.in) + assert.Equal(t, c.expectMapped, mapped) + assert.Equal(t, c.expectedKnown, known) + }) + } +} + +func TestMapStatusDoesNotPanic(t *testing.T) { + assert.NotPanics(t, func() { + _, _ = mapStatus("brand-new-status-rootly-invented-yesterday") + }) +} + +func TestMapSeverityToPriority(t *testing.T) { + cases := []struct { + in string + expected string + }{ + {"sev0", "CRITICAL"}, + {"SEV0", "CRITICAL"}, + {"Sev0", "CRITICAL"}, + {"sev1", "HIGH"}, + {"SEV1", "HIGH"}, + {"sev2", "MEDIUM"}, + {"sev3", "LOW"}, + {"sev4", "LOW"}, + {"sev5", "sev5"}, + {"critical-ish", "critical-ish"}, + {"", ""}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + assert.Equal(t, c.expected, mapSeverityToPriority(c.in)) + }) + } +} + +func TestComputeLeadTime_Resolved(t *testing.T) { + started := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + resolved := time.Date(2026, 5, 10, 11, 30, 0, 0, time.UTC) + updated := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(started, &resolved, updated, "resolved") + require.NotNil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, uint(90), *leadTime) + assert.Equal(t, resolved, *resolutionDate) +} + +func TestComputeLeadTime_Unresolved(t *testing.T) { + started := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + updated := time.Date(2026, 5, 10, 12, 0, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(started, nil, updated, "started") + assert.Nil(t, leadTime) + assert.Nil(t, resolutionDate) +} + +func TestComputeLeadTime_CompletedFallsBackToUpdated(t *testing.T) { + started := time.Date(2026, 3, 27, 20, 0, 0, 0, time.UTC) + updated := time.Date(2026, 3, 31, 14, 0, 0, 0, time.UTC) + leadTime, resolutionDate := computeLeadTime(started, nil, updated, "completed") + require.NotNil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, updated, *resolutionDate) +} + +func TestComputeLeadTime_ZeroDuration(t *testing.T) { + started := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + resolved := started + updated := started + leadTime, resolutionDate := computeLeadTime(started, &resolved, updated, "resolved") + require.NotNil(t, leadTime) + require.NotNil(t, resolutionDate) + assert.Equal(t, uint(0), *leadTime) +} + +func TestComputeLeadTime_ResolvedBeforeStarted(t *testing.T) { + started := time.Date(2026, 5, 10, 11, 0, 0, 0, time.UTC) + resolved := time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC) + updated := started + leadTime, resolutionDate := computeLeadTime(started, &resolved, updated, "resolved") + assert.Nil(t, leadTime) + assert.Nil(t, resolutionDate) +} + +func TestIssueKeyFor(t *testing.T) { + cases := []struct { + name string + incident models.Incident + expected string + }{ + {"positive sequential id", models.Incident{Number: 42, Id: "inc_abc"}, "42"}, + {"zero sequential id falls back to slug", models.Incident{Number: 0, Id: "inc_abc"}, "inc_abc"}, + {"negative sequential id falls back to slug", models.Incident{Number: -1, Id: "inc_abc"}, "inc_abc"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.expected, issueKeyFor(&c.incident)) + }) + } +} + +func TestAssigneeDedup(t *testing.T) { + cases := []struct { + name string + incident models.Incident + expected []string + }{ + { + name: "all roles empty", + incident: models.Incident{}, + expected: []string{}, + }, + { + name: "single creator", + incident: models.Incident{CreatorUserId: "u1"}, + expected: []string{"u1"}, + }, + { + name: "same user in creator and resolver", + incident: models.Incident{ + CreatorUserId: "u1", + ResolvedByUserId: "u1", + }, + expected: []string{"u1"}, + }, + { + name: "distinct users across all roles", + incident: models.Incident{ + CreatorUserId: "u1", + StartedByUserId: "u2", + MitigatedByUserId: "u3", + ResolvedByUserId: "u4", + ClosedByUserId: "u5", + }, + expected: []string{"u1", "u2", "u3", "u4", "u5"}, + }, + { + name: "empty interleaved with populated", + incident: models.Incident{ + CreatorUserId: "u1", + StartedByUserId: "", + MitigatedByUserId: "u2", + ResolvedByUserId: "", + ClosedByUserId: "u1", + }, + expected: []string{"u1", "u2"}, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + seen := map[string]bool{} + var got []string + for _, uid := range c.incident.RoleUserIds() { + if uid == "" || seen[uid] { + continue + } + seen[uid] = true + got = append(got, uid) + } + if len(c.expected) == 0 { + assert.Empty(t, got) + } else { + assert.Equal(t, c.expected, got) + } + }) + } +} + +func TestMapStatus_MitigatedIsKnown(t *testing.T) { + mapped, known := mapStatus("mitigated") + assert.Equal(t, ticket.IN_PROGRESS, mapped) + assert.True(t, known) + mapped, known = mapStatus("something-else") + assert.Equal(t, ticket.IN_PROGRESS, mapped) + assert.False(t, known) +} diff --git a/backend/plugins/rootly/tasks/incidents_extractor.go b/backend/plugins/rootly/tasks/incidents_extractor.go new file mode 100644 index 00000000000..3e812e2c79a --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_extractor.go @@ -0,0 +1,160 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/apache/incubator-devlake/plugins/rootly/models/raw" +) + +var _ plugin.SubTaskEntryPoint = ExtractIncidents + +var ExtractIncidentsMeta = plugin.SubTaskMeta{ + Name: "extractIncidents", + EntryPoint: ExtractIncidents, + EnabledByDefault: true, + Description: "Extract Rootly incidents", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{models.Incident{}.TableName(), models.User{}.TableName()}, +} + +func ExtractIncidents(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*RootlyTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_INCIDENTS_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + return extractRootlyIncident(row.Data, data.Options) + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} + +func extractRootlyIncident(rawData []byte, op *RootlyOptions) ([]interface{}, errors.Error) { + rawIncident := &raw.Incident{} + if err := errors.Convert(json.Unmarshal(rawData, rawIncident)); err != nil { + return nil, err + } + + // Safety net: filter[service_ids] in the collector is the primary + // scope filter, but a regression there would let multi-service + // incidents leak into a wrong scope's tool table. When ServiceId is + // empty we are collecting all incidents globally, so skip this check. + if op.ServiceId != "" { + if services := rawIncident.Relationships.Services.Data; len(services) > 0 && !containsServiceId(services, op.ServiceId) { + return nil, nil + } + } + + if rawIncident.Attributes.StartedAt.IsZero() { + return nil, errors.Default.New("rootly incident missing started_at") + } + + incident := &models.Incident{ + ConnectionId: op.ConnectionId, + Id: rawIncident.Id, + Number: resolve(rawIncident.Attributes.SequentialId), + ServiceId: op.ServiceId, + Url: resolve(rawIncident.Attributes.Url), + Title: rawIncident.Attributes.Title, + Summary: resolve(rawIncident.Attributes.Summary), + Status: rawIncident.Attributes.Status, + Severity: resolveSeverity(rawIncident.Attributes.Severity), + StartedDate: rawIncident.Attributes.StartedAt, + AcknowledgedDate: rawIncident.Attributes.AcknowledgedAt, + MitigatedDate: rawIncident.Attributes.MitigatedAt, + ResolvedDate: rawIncident.Attributes.ResolvedAt, + UpdatedDate: rawIncident.Attributes.UpdatedAt, + } + + results := []interface{}{incident} + seen := map[string]bool{} + addUser := func(u *raw.UserEnvelope, setRoleId func(string)) { + if u == nil || u.Data.Id == "" { + return + } + setRoleId(u.Data.Id) + if seen[u.Data.Id] { + return + } + seen[u.Data.Id] = true + name := pickUserName(u.Data.Attributes) + // Skip rows with no useful data so a sibling scope task that has + // fuller data for the same user doesn't get overwritten with blanks. + if name == "" && u.Data.Attributes.Email == "" { + return + } + results = append(results, &models.User{ + ConnectionId: op.ConnectionId, + Id: u.Data.Id, + Email: u.Data.Attributes.Email, + Name: name, + }) + } + addUser(rawIncident.Attributes.User, func(id string) { incident.CreatorUserId = id }) + addUser(rawIncident.Attributes.StartedBy, func(id string) { incident.StartedByUserId = id }) + addUser(rawIncident.Attributes.MitigatedBy, func(id string) { incident.MitigatedByUserId = id }) + addUser(rawIncident.Attributes.ResolvedBy, func(id string) { incident.ResolvedByUserId = id }) + addUser(rawIncident.Attributes.ClosedBy, func(id string) { incident.ClosedByUserId = id }) + + return results, nil +} + +func pickUserName(u raw.UserAttributes) string { + if u.FullName != "" { + return u.FullName + } + if u.Name != "" { + return u.Name + } + return u.Email +} + +func containsServiceId(services []raw.ServiceRef, serviceId string) bool { + for _, s := range services { + if s.Id == serviceId { + return true + } + } + return false +} + +func resolveSeverity(s *raw.SeverityEnvelope) string { + if s == nil { + return "" + } + return s.Data.Attributes.Slug +} + +func resolve[T any](t *T) T { + if t == nil { + return *new(T) + } + return *t +} diff --git a/backend/plugins/rootly/tasks/incidents_extractor_test.go b/backend/plugins/rootly/tasks/incidents_extractor_test.go new file mode 100644 index 00000000000..cba3410116e --- /dev/null +++ b/backend/plugins/rootly/tasks/incidents_extractor_test.go @@ -0,0 +1,382 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +const baseHappyPathActive = `{ + "id": "inc_01", + "type": "incidents", + "attributes": { + "sequential_id": 42, + "title": "db outage", + "summary": "replica lag blew past threshold", + "url": "https://rootly.example.com/incidents/inc_01", + "status": "started", + "severity": {"data": {"id": "sev-uuid-1", "type": "severities", "attributes": {"slug": "sev1", "name": "SEV1", "severity": "high"}}}, + "started_at": "2026-05-10T10:00:00Z", + "updated_at": "2026-05-10T10:05:00Z", + "user": {"data": {"id": "usr_100", "type": "users", "attributes": {"name": "Reporter One", "full_name": "Reporter One", "email": "reporter@example.com"}}} + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } +}` + +func newTestOptions() *RootlyOptions { + return &RootlyOptions{ + ConnectionId: 7, + ServiceId: "svc_02", + } +} + +func collectUsers(results []interface{}) []*models.User { + users := []*models.User{} + for _, r := range results { + if u, ok := r.(*models.User); ok { + users = append(users, u) + } + } + return users +} + +func TestExtractRootlyIncident_HappyPathActive(t *testing.T) { + op := newTestOptions() + results, err := extractRootlyIncident([]byte(baseHappyPathActive), op) + require.NoError(t, err) + require.Len(t, results, 2) + + incident, ok := results[0].(*models.Incident) + require.True(t, ok, "first result should be *models.Incident") + assert.Equal(t, uint64(7), incident.ConnectionId) + assert.Equal(t, "inc_01", incident.Id) + assert.Equal(t, 42, incident.Number) + assert.Equal(t, "svc_02", incident.ServiceId) + assert.Equal(t, "db outage", incident.Title) + assert.Equal(t, "replica lag blew past threshold", incident.Summary) + assert.Equal(t, "https://rootly.example.com/incidents/inc_01", incident.Url) + assert.Equal(t, "started", incident.Status) + assert.Equal(t, "sev1", incident.Severity) + assert.Equal(t, time.Date(2026, 5, 10, 10, 0, 0, 0, time.UTC), incident.StartedDate) + assert.Nil(t, incident.AcknowledgedDate) + assert.Nil(t, incident.MitigatedDate) + assert.Nil(t, incident.ResolvedDate) + assert.Equal(t, time.Date(2026, 5, 10, 10, 5, 0, 0, time.UTC), incident.UpdatedDate) + + assert.Equal(t, "usr_100", incident.CreatorUserId) + assert.Empty(t, incident.StartedByUserId) + assert.Empty(t, incident.MitigatedByUserId) + assert.Empty(t, incident.ResolvedByUserId) + assert.Empty(t, incident.ClosedByUserId) + + users := collectUsers(results) + require.Len(t, users, 1) + assert.Equal(t, "usr_100", users[0].Id) + assert.Equal(t, uint64(7), users[0].ConnectionId) + assert.Equal(t, "Reporter One", users[0].Name) + assert.Equal(t, "reporter@example.com", users[0].Email) +} + +func TestExtractRootlyIncident_Resolved(t *testing.T) { + raw := []byte(`{ + "id": "inc_02", + "type": "incidents", + "attributes": { + "sequential_id": 43, + "title": "cache cleared", + "status": "resolved", + "severity": {"data": {"id": "sev-uuid-3", "type": "severities", "attributes": {"slug": "sev3", "severity": "low"}}}, + "started_at": "2026-05-09T08:00:00Z", + "acknowledged_at": "2026-05-09T08:05:00Z", + "mitigated_at": "2026-05-09T08:30:00Z", + "resolved_at": "2026-05-09T09:00:00Z", + "updated_at": "2026-05-09T09:01:00Z", + "user": {"data": {"id": "usr_100", "type": "users", "attributes": {"full_name": "Reporter One"}}}, + "resolved_by": {"data": {"id": "usr_200", "type": "users", "attributes": {"full_name": "Resolver Two"}}} + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 3) + + incident := results[0].(*models.Incident) + require.NotNil(t, incident.AcknowledgedDate) + require.NotNil(t, incident.MitigatedDate) + require.NotNil(t, incident.ResolvedDate) + assert.Equal(t, "resolved", incident.Status) + assert.Equal(t, time.Date(2026, 5, 9, 9, 0, 0, 0, time.UTC), *incident.ResolvedDate) + assert.Equal(t, time.Date(2026, 5, 9, 8, 30, 0, 0, time.UTC), *incident.MitigatedDate) + assert.Equal(t, time.Date(2026, 5, 9, 8, 5, 0, 0, time.UTC), *incident.AcknowledgedDate) + + assert.Equal(t, "usr_100", incident.CreatorUserId) + assert.Equal(t, "usr_200", incident.ResolvedByUserId) + + users := collectUsers(results) + require.Len(t, users, 2) + ids := map[string]string{} + for _, u := range users { + ids[u.Id] = u.Name + } + assert.Equal(t, "Reporter One", ids["usr_100"]) + assert.Equal(t, "Resolver Two", ids["usr_200"]) +} + +func TestExtractRootlyIncident_MissingOptionalTimestamps(t *testing.T) { + raw := []byte(`{ + "id": "inc_03", + "type": "incidents", + "attributes": { + "sequential_id": 44, + "title": "ongoing issue", + "status": "started", + "started_at": "2026-05-10T12:00:00Z", + "updated_at": "2026-05-10T12:05:00Z" + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Nil(t, incident.MitigatedDate) + assert.Nil(t, incident.ResolvedDate) + assert.Nil(t, incident.AcknowledgedDate) +} + +func TestExtractRootlyIncident_NullSeverity(t *testing.T) { + raw := []byte(`{ + "id": "inc_04", + "type": "incidents", + "attributes": { + "sequential_id": 45, + "title": "no sev yet", + "status": "mitigated", + "severity": null, + "started_at": "2026-05-10T14:00:00Z", + "updated_at": "2026-05-10T14:05:00Z" + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, "", incident.Severity) +} + +func TestExtractRootlyIncident_NoRolesFilled(t *testing.T) { + raw := []byte(`{ + "id": "inc_05", + "type": "incidents", + "attributes": { + "sequential_id": 46, + "title": "ghost incident", + "status": "started", + "started_at": "2026-05-10T15:00:00Z", + "updated_at": "2026-05-10T15:05:00Z", + "user": null, + "started_by": null, + "mitigated_by": null, + "resolved_by": null, + "closed_by": null + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Empty(t, incident.CreatorUserId) + assert.Empty(t, incident.StartedByUserId) + assert.Empty(t, incident.MitigatedByUserId) + assert.Empty(t, incident.ResolvedByUserId) + assert.Empty(t, incident.ClosedByUserId) + assert.Empty(t, collectUsers(results)) +} + +func TestExtractRootlyIncident_SameUserInMultipleRoles(t *testing.T) { + raw := []byte(`{ + "id": "inc_dup", + "type": "incidents", + "attributes": { + "sequential_id": 47, + "title": "solo fire", + "status": "resolved", + "started_at": "2026-05-10T16:00:00Z", + "resolved_at": "2026-05-10T16:30:00Z", + "updated_at": "2026-05-10T16:31:00Z", + "user": {"data": {"id": "usr_100", "type": "users", "attributes": {"full_name": "Solo Operator"}}}, + "resolved_by": {"data": {"id": "usr_100", "type": "users", "attributes": {"full_name": "Solo Operator"}}} + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 2, "one incident + one deduped user") + + incident := results[0].(*models.Incident) + assert.Equal(t, "usr_100", incident.CreatorUserId) + assert.Equal(t, "usr_100", incident.ResolvedByUserId) + + users := collectUsers(results) + require.Len(t, users, 1) + assert.Equal(t, "usr_100", users[0].Id) + assert.Equal(t, "Solo Operator", users[0].Name) +} + +func TestExtractRootlyIncident_UserNamePreference(t *testing.T) { + raw := []byte(`{ + "id": "inc_names", + "type": "incidents", + "attributes": { + "sequential_id": 48, + "title": "name preference", + "status": "started", + "started_at": "2026-05-10T17:00:00Z", + "updated_at": "2026-05-10T17:05:00Z", + "user": {"data": {"id": "usr_full", "type": "users", "attributes": {"full_name": "Full Name", "name": "Ignored", "email": "ignored@example.com"}}}, + "started_by": {"data": {"id": "usr_short", "type": "users", "attributes": {"name": "Short Name", "email": "ignored@example.com"}}}, + "resolved_by": {"data": {"id": "usr_mail", "type": "users", "attributes": {"email": "fallback@example.com"}}} + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + + users := collectUsers(results) + require.Len(t, users, 3) + byId := map[string]*models.User{} + for _, u := range users { + byId[u.Id] = u + } + require.Contains(t, byId, "usr_full") + require.Contains(t, byId, "usr_short") + require.Contains(t, byId, "usr_mail") + assert.Equal(t, "Full Name", byId["usr_full"].Name) + assert.Equal(t, "Short Name", byId["usr_short"].Name) + assert.Equal(t, "fallback@example.com", byId["usr_mail"].Name) +} + +func TestExtractRootlyIncident_WrongServiceSkipped(t *testing.T) { + raw := []byte(`{ + "id": "inc_wrong_svc", + "type": "incidents", + "attributes": { + "sequential_id": 49, + "title": "other service", + "status": "started", + "started_at": "2026-05-10T18:00:00Z", + "updated_at": "2026-05-10T18:05:00Z" + }, + "relationships": { + "services": {"data": [{"id": "svc_99", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + assert.Empty(t, results, "incident for unrelated service should produce no rows") +} + +func TestExtractRootlyIncident_EmptyServicesAccepted(t *testing.T) { + raw := []byte(`{ + "id": "inc_no_svc", + "type": "incidents", + "attributes": { + "sequential_id": 50, + "title": "services omitted", + "status": "started", + "started_at": "2026-05-10T19:00:00Z", + "updated_at": "2026-05-10T19:05:00Z" + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, "svc_02", incident.ServiceId) +} + +func TestExtractRootlyIncident_MissingStartedAtReturnsError(t *testing.T) { + raw := []byte(`{ + "id": "inc_bad", + "type": "incidents", + "attributes": { + "sequential_id": 51, + "title": "bad row", + "status": "started", + "updated_at": "2026-05-10T20:05:00Z" + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + _, err := extractRootlyIncident(raw, op) + assert.Error(t, err) +} + +func TestExtractRootlyIncident_MissingSequentialId(t *testing.T) { + raw := []byte(`{ + "id": "inc_no_num", + "type": "incidents", + "attributes": { + "title": "no sequential id", + "status": "started", + "started_at": "2026-05-10T21:00:00Z", + "updated_at": "2026-05-10T21:05:00Z" + }, + "relationships": { + "services": {"data": [{"id": "svc_02", "type": "services"}]} + } + }`) + op := newTestOptions() + results, err := extractRootlyIncident(raw, op) + require.NoError(t, err) + require.Len(t, results, 1) + incident := results[0].(*models.Incident) + assert.Equal(t, 0, incident.Number) +} diff --git a/backend/plugins/rootly/tasks/service_converter.go b/backend/plugins/rootly/tasks/service_converter.go new file mode 100644 index 00000000000..be5e890ff85 --- /dev/null +++ b/backend/plugins/rootly/tasks/service_converter.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +var ConvertServicesMeta = plugin.SubTaskMeta{ + Name: "convertServices", + EntryPoint: ConvertServices, + EnabledByDefault: true, + Description: "Convert Rootly services", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertServices(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*RootlyTaskData) + rawDataSubTaskArgs := &helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_SERVICES_TABLE, + } + clauses := []dal.Clause{ + dal.Select("services.*"), + dal.From("_tool_rootly_services services"), + dal.Where("id = ? and connection_id = ?", data.Options.ServiceId, data.Options.ConnectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.Service{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + service := inputRow.(*models.Service) + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: didgen.NewDomainIdGenerator(service).Generate(service.ConnectionId, service.Id), + }, + Name: service.Name, + Url: service.Url, + } + return []interface{}{ + domainBoard, + }, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/rootly/tasks/services_collector.go b/backend/plugins/rootly/tasks/services_collector.go new file mode 100644 index 00000000000..9800b47e6b4 --- /dev/null +++ b/backend/plugins/rootly/tasks/services_collector.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_SERVICES_TABLE = "rootly_services" + +// singleServiceResponse is the JSON:API envelope returned by +// GET /services/{id}. Unlike list responses, `data` is an object, +// not an array. +type singleServiceResponse struct { + Data json.RawMessage `json:"data"` +} + +var _ plugin.SubTaskEntryPoint = CollectServices + +var CollectServicesMeta = plugin.SubTaskMeta{ + Name: "collectServices", + EntryPoint: CollectServices, + EnabledByDefault: true, + Description: "Collect Rootly services", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{RAW_SERVICES_TABLE}, +} + +func CollectServices(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*RootlyTaskData) + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_SERVICES_TABLE, + }, + ApiClient: data.Client, + UrlTemplate: "services/{{ .Params.ScopeId }}", + Query: nil, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + rawResult := singleServiceResponse{} + err := api.UnmarshalResponse(res, &rawResult) + if err != nil { + return nil, err + } + if len(rawResult.Data) == 0 { + return []json.RawMessage{}, nil + } + return []json.RawMessage{rawResult.Data}, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/rootly/tasks/services_extractor.go b/backend/plugins/rootly/tasks/services_extractor.go new file mode 100644 index 00000000000..bc5cea0d230 --- /dev/null +++ b/backend/plugins/rootly/tasks/services_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" + "github.com/apache/incubator-devlake/plugins/rootly/models/raw" +) + +var _ plugin.SubTaskEntryPoint = ExtractServices + +var ExtractServicesMeta = plugin.SubTaskMeta{ + Name: "extractServices", + EntryPoint: ExtractServices, + EnabledByDefault: true, + Description: "Extract Rootly services", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + ProductTables: []string{models.Service{}.TableName()}, +} + +func ExtractServices(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*RootlyTaskData) + db := taskCtx.GetDal() + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_SERVICES_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + rawService := &raw.Service{} + if err := errors.Convert(json.Unmarshal(row.Data, rawService)); err != nil { + return nil, err + } + url := "" + if rawService.Attributes.HtmlUrl != nil { + url = *rawService.Attributes.HtmlUrl + } + service := &models.Service{ + Id: rawService.Id, + Name: rawService.Attributes.Name, + Url: url, + } + service.ConnectionId = data.Options.ConnectionId + // Preserve operator-set ScopeConfigId across re-collections. + existing := &models.Service{} + if err := db.First(existing, dal.Where("connection_id = ? AND id = ?", data.Options.ConnectionId, rawService.Id)); err == nil { + service.ScopeConfigId = existing.ScopeConfigId + } + return []interface{}{service}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/rootly/tasks/task_data.go b/backend/plugins/rootly/tasks/task_data.go new file mode 100644 index 00000000000..c530d1d1706 --- /dev/null +++ b/backend/plugins/rootly/tasks/task_data.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/rootly/models" +) + +type RootlyOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + ServiceId string `json:"serviceId,omitempty" mapstructure:"serviceId,omitempty"` + ServiceName string `json:"serviceName,omitempty" mapstructure:"serviceName,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"` + ScopeConfig *models.RootlyScopeConfig `json:"scopeConfig,omitempty" mapstructure:"scopeConfig,omitempty"` +} + +type RootlyTaskData struct { + Options *RootlyOptions + Client api.RateLimitedApiClient +} + +func (p *RootlyOptions) GetParams() any { + scopeId := p.ServiceId + if scopeId == "" { + scopeId = "all" + } + return models.RootlyParams{ + ConnectionId: p.ConnectionId, + ScopeId: scopeId, + } +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*RootlyOptions, errors.Error) { + op, err := DecodeTaskOptions(options) + if err != nil { + return nil, err + } + err = ValidateTaskOptions(op) + if err != nil { + return nil, err + } + return op, nil +} + +func DecodeTaskOptions(options map[string]interface{}) (*RootlyOptions, errors.Error) { + var op RootlyOptions + err := api.Decode(options, &op, nil) + if err != nil { + return nil, err + } + return &op, nil +} + +func ValidateTaskOptions(op *RootlyOptions) errors.Error { + if op.ConnectionId == 0 { + return errors.BadInput.New("connectionId is invalid") + } + return nil +} diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go b/backend/plugins/sonarqube/api/blueprint_v200.go index e4804534b5d..ba3b5eedf64 100644 --- a/backend/plugins/sonarqube/api/blueprint_v200.go +++ b/backend/plugins/sonarqube/api/blueprint_v200.go @@ -130,8 +130,11 @@ func GetApiProject( Data []models.SonarqubeApiProject `json:"components"` } query := url.Values{} - query.Set("q", projectKey) - res, err := apiClient.Get("projects/search", query, nil) + query.Set("p", "1") + query.Set("ps", "100") + query.Set("filter", sonarqubeSearchProjectsQueryFilter(projectKey)) + // Use components/search_projects for consistency and normal-token (Browse) support. + res, err := apiClient.Get("components/search_projects", query, nil) if err != nil { return nil, err } @@ -142,6 +145,11 @@ func GetApiProject( if err != nil { return nil, err } + for i := range resData.Data { + if resData.Data[i].ProjectKey == projectKey { + return &resData.Data[i], nil + } + } if len(resData.Data) > 0 { return &resData.Data[0], nil } diff --git a/backend/plugins/sonarqube/api/connection_api.go b/backend/plugins/sonarqube/api/connection_api.go index feb1905a63d..d387e5d460a 100644 --- a/backend/plugins/sonarqube/api/connection_api.go +++ b/backend/plugins/sonarqube/api/connection_api.go @@ -44,6 +44,9 @@ func testConnection(ctx context.Context, connection models.SonarqubeConn) (*plug return nil, errors.Default.Wrap(err, "error validating target") } } + if err := connection.ValidateUserTokenPrefix(); err != nil { + return nil, err + } apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) if err != nil { return nil, err diff --git a/backend/plugins/sonarqube/api/remote_api.go b/backend/plugins/sonarqube/api/remote_api.go index 704cdabf5e7..02ca0f35f31 100644 --- a/backend/plugins/sonarqube/api/remote_api.go +++ b/backend/plugins/sonarqube/api/remote_api.go @@ -20,6 +20,7 @@ package api import ( "fmt" "net/url" + "strings" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" @@ -33,6 +34,16 @@ type SonarqubeRemotePagination struct { PageSize int `json:"ps"` } +// sonarqubeSearchProjectsQueryFilter builds the "filter" query value for +// api/components/search_projects. SonarQube expects text search via the filter +// language (e.g. filter=query = "term"), not a bare "q" parameter; see +// SearchProjectsAction in SonarQube. +func sonarqubeSearchProjectsQueryFilter(term string) string { + term = strings.TrimSpace(term) + term = strings.ReplaceAll(term, `"`, "") + return fmt.Sprintf(`query = "%s"`, term) +} + func querySonarqubeProjects( apiClient plugin.ApiClient, keyword string, @@ -48,11 +59,16 @@ func querySonarqubeProjects( if page.Page == 0 { page.Page = 1 } - res, err := apiClient.Get("projects/search", url.Values{ + // Use components/search_projects so non-admin (Browse) tokens can list projects. + q := url.Values{ "p": {fmt.Sprintf("%v", page.Page)}, "ps": {fmt.Sprintf("%v", page.PageSize)}, - "q": {keyword}, - }, nil) + } + keyword = strings.TrimSpace(keyword) + if keyword != "" { + q.Set("filter", sonarqubeSearchProjectsQueryFilter(keyword)) + } + res, err := apiClient.Get("components/search_projects", q, nil) if err != nil { return } diff --git a/backend/plugins/sonarqube/api/remote_api_test.go b/backend/plugins/sonarqube/api/remote_api_test.go new file mode 100644 index 00000000000..a3b290dbff4 --- /dev/null +++ b/backend/plugins/sonarqube/api/remote_api_test.go @@ -0,0 +1,34 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import "testing" + +func TestSonarqubeSearchProjectsQueryFilter(t *testing.T) { + t.Parallel() + if g, w := sonarqubeSearchProjectsQueryFilter("iap"), `query = "iap"`; g != w { + t.Fatalf("got %q want %q", g, w) + } + if g, w := sonarqubeSearchProjectsQueryFilter(" my term "), `query = "my term"`; g != w { + t.Fatalf("got %q want %q", g, w) + } + // double quotes are stripped to avoid breaking the filter string + if g, w := sonarqubeSearchProjectsQueryFilter(`a"x`), `query = "ax"`; g != w { + t.Fatalf("got %q want %q", g, w) + } +} diff --git a/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go b/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go new file mode 100644 index 00000000000..8076fa3a859 --- /dev/null +++ b/backend/plugins/sonarqube/e2e/issue_code_block_long_component_test.go @@ -0,0 +1,199 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "strings" + "testing" + "time" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + coremigrations "github.com/apache/incubator-devlake/core/models/migrationscripts" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + implcontext "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/plugins/sonarqube/impl" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" + sonarqubemigrations "github.com/apache/incubator-devlake/plugins/sonarqube/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/sonarqube/tasks" + "github.com/stretchr/testify/require" +) + +type sonarqubeIssueCodeBlockBeforeText struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey"` + IssueKey string `gorm:"index"` + Component string `gorm:"index;type:varchar(500)"` + StartLine int + EndLine int + StartOffset int + EndOffset int + Msg string + common.NoPKModel +} + +func (sonarqubeIssueCodeBlockBeforeText) TableName() string { + return "_tool_sonarqube_issue_code_blocks" +} + +type cqIssueCodeBlockBeforeText struct { + domainlayer.DomainEntity + IssueKey string `json:"key" gorm:"index"` + Component string `gorm:"index"` + StartLine int + EndLine int + StartOffset int + EndOffset int + Msg string +} + +func (cqIssueCodeBlockBeforeText) TableName() string { + return "cq_issue_code_blocks" +} + +func TestSonarqubeIssueCodeBlockLongComponent(t *testing.T) { + var sonarqube impl.Sonarqube + dataflowTester := e2ehelper.NewDataFlowTester(t, "sonarqube", sonarqube) + dataflowTester.FlushTabler(&models.SonarqubeIssue{}) + require.NoError(t, dataflowTester.Db.Migrator().DropTable( + &sonarqubeIssueCodeBlockBeforeText{}, + &cqIssueCodeBlockBeforeText{}, + )) + require.NoError(t, dataflowTester.Db.AutoMigrate( + &sonarqubeIssueCodeBlockBeforeText{}, + &cqIssueCodeBlockBeforeText{}, + )) + + existingComponent := "existing:component" + require.NoError(t, dataflowTester.Db.Create(&sonarqubeIssueCodeBlockBeforeText{ + ConnectionId: 1, + Id: "existing-tool-block", + IssueKey: "existing-issue", + Component: existingComponent, + }).Error) + require.NoError(t, dataflowTester.Db.Create(&cqIssueCodeBlockBeforeText{ + DomainEntity: domainlayer.DomainEntity{Id: "existing-domain-block"}, + IssueKey: "existing-domain-issue", + Component: existingComponent, + }).Error) + + basicRes := implcontext.NewDefaultBasicRes(dataflowTester.Cfg, dataflowTester.Log, dataflowTester.Dal) + runMigration(t, coremigrations.All(), "change cq_issue_code_blocks.component type to text", basicRes) + runMigration(t, sonarqubemigrations.All(), "change _tool_sonarqube_issue_code_blocks.component type to text", basicRes) + assertTextColumnWithoutIndex(t, dataflowTester, "cq_issue_code_blocks") + assertTextColumnWithoutIndex(t, dataflowTester, "_tool_sonarqube_issue_code_blocks") + + var migratedToolBlock sonarqubeIssueCodeBlockBeforeText + require.NoError(t, dataflowTester.Db.First(&migratedToolBlock, "id = ?", "existing-tool-block").Error) + require.Equal(t, existingComponent, migratedToolBlock.Component) + var migratedDomainBlock cqIssueCodeBlockBeforeText + require.NoError(t, dataflowTester.Db.First(&migratedDomainBlock, "id = ?", "existing-domain-block").Error) + require.Equal(t, existingComponent, migratedDomainBlock.Component) + require.NoError(t, dataflowTester.Db.Delete(&migratedToolBlock).Error) + require.NoError(t, dataflowTester.Db.Delete(&migratedDomainBlock).Error) + + longComponent256 := "project:" + strings.Repeat("a", 256) + longComponent500 := "project:" + strings.Repeat("b", 500) + require.Greater(t, len(longComponent256), 256) + require.Greater(t, len(longComponent500), 500) + + issueKey := "TEST-LONG-COMPONENT-ISSUE" + projectKey := "test-long-component-project" + result := dataflowTester.Db.Create(&models.SonarqubeIssue{ + ConnectionId: 1, + IssueKey: issueKey, + ProjectKey: projectKey, + Component: longComponent500, + Rule: "java:S3776", + Severity: "CRITICAL", + }) + require.NoError(t, result.Error) + + codeBlocks := []*models.SonarqubeIssueCodeBlock{ + { + ConnectionId: 1, + Id: "test-long-component-block-256", + IssueKey: issueKey, + Component: longComponent256, + Msg: "component longer than 256 characters", + }, + { + ConnectionId: 1, + Id: "test-long-component-block-500", + IssueKey: issueKey, + Component: longComponent500, + Msg: "component longer than 500 characters", + }, + } + for _, block := range codeBlocks { + require.NoError(t, dataflowTester.Db.Create(block).Error) + } + + dataflowTester.Subtask(tasks.ConvertIssueCodeBlocksMeta, &tasks.SonarqubeTaskData{ + Options: &tasks.SonarqubeOptions{ + ConnectionId: 1, + ProjectKey: projectKey, + }, + TaskStartTime: time.Now(), + }) + + var domainBlocks []codequality.CqIssueCodeBlock + require.NoError(t, dataflowTester.Db.Find(&domainBlocks).Error) + require.Len(t, domainBlocks, 2) + require.ElementsMatch(t, + []string{longComponent256, longComponent500}, + []string{domainBlocks[0].Component, domainBlocks[1].Component}, + ) + + var toolBlocks []models.SonarqubeIssueCodeBlock + require.NoError(t, dataflowTester.Db.Where( + "connection_id = ? AND issue_key = ?", 1, issueKey, + ).Find(&toolBlocks).Error) + require.Len(t, toolBlocks, 2) + require.ElementsMatch(t, + []string{longComponent256, longComponent500}, + []string{toolBlocks[0].Component, toolBlocks[1].Component}, + ) +} + +func runMigration(t *testing.T, scripts []plugin.MigrationScript, name string, basicRes *implcontext.DefaultBasicRes) { + t.Helper() + for _, script := range scripts { + if script.Name() == name { + require.NoError(t, script.Up(basicRes)) + return + } + } + require.Fail(t, "migration is not registered", name) +} + +func assertTextColumnWithoutIndex(t *testing.T, dataflowTester *e2ehelper.DataFlowTester, table string) { + t.Helper() + columnTypes, err := dataflowTester.Db.Migrator().ColumnTypes(table) + require.NoError(t, err) + for _, columnType := range columnTypes { + if columnType.Name() == "component" { + require.Contains(t, strings.ToLower(columnType.DatabaseTypeName()), "text") + require.False(t, dataflowTester.Db.Migrator().HasIndex(table, "idx_"+table+"_component")) + return + } + } + require.Fail(t, "component column not found", table) +} diff --git a/backend/plugins/sonarqube/impl/impl.go b/backend/plugins/sonarqube/impl/impl.go index 5c1a774867b..85ee668ee31 100644 --- a/backend/plugins/sonarqube/impl/impl.go +++ b/backend/plugins/sonarqube/impl/impl.go @@ -86,6 +86,8 @@ func (p Sonarqube) GetTablesInfo() []dal.Tabler { &models.SonarqubeFileMetrics{}, &models.SonarqubeAccount{}, &models.SonarqubeScopeConfig{}, + &models.SonarqubeProjectMetricsHistory{}, + &models.SonarqubeProjectAnalysis{}, } } @@ -108,6 +110,11 @@ func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta { tasks.ConvertHotspotsMeta, tasks.ConvertFileMetricsMeta, tasks.ConvertAccountsMeta, + tasks.CollectProjectMetricsHistoryMeta, + tasks.ExtractProjectMetricsHistoryMeta, + tasks.CollectProjectAnalysesMeta, + tasks.ExtractProjectAnalysesMeta, + tasks.ConvertProjectMetricsHistoryMeta, } } @@ -136,6 +143,7 @@ func (p Sonarqube) PrepareTaskData(taskCtx plugin.TaskContext, options map[strin Options: op, ApiClient: apiClient, TaskStartTime: time.Now(), + IsCloud: connection.IsCloud(), } // even we have project in _tool_sonaqube_projects, we still need to collect project to update LastAnalysisDate var apiProject *models.SonarqubeApiProject diff --git a/backend/plugins/sonarqube/models/connection.go b/backend/plugins/sonarqube/models/connection.go index 66c32f57c49..0bbe124d592 100644 --- a/backend/plugins/sonarqube/models/connection.go +++ b/backend/plugins/sonarqube/models/connection.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "net/http" + "strings" "github.com/apache/incubator-devlake/core/utils" @@ -29,6 +30,12 @@ import ( helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) +const ( + userTokenPrefix = "squ_" + globalAnalysisTokenPrefix = "sqa_" + projectAnalysisTokenPrefix = "sqp_" +) + type SonarqubeAccessToken helper.AccessToken // SetupAuthentication sets up the HTTP Request Authentication @@ -92,7 +99,40 @@ func (connection *SonarqubeConnection) MergeFromRequest(target *SonarqubeConnect return nil } -func (connection *SonarqubeConnection) IsCloud() bool { +// ValidateUserTokenPrefix ensures the token is a SonarQube User token on Server +// instances. Global (sqa_) and Project (sqp_) analysis tokens authenticate but +// cannot call read APIs such as measures/component_tree. +func (connection SonarqubeConn) ValidateUserTokenPrefix() errors.Error { + if connection.IsCloud() { + return nil + } + token := strings.TrimSpace(connection.Token) + if token == "" { + return errors.BadInput.New("token is required") + } + switch { + case strings.HasPrefix(token, globalAnalysisTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix). " + + "Global Analysis tokens (sqa_) can push scan results but cannot read project metrics via the Web API. " + + "Create a User token under My Account > Security in SonarQube.", + ) + case strings.HasPrefix(token, projectAnalysisTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix). " + + "Project Analysis tokens (sqp_) can push scan results but cannot read project metrics via the Web API. " + + "Create a User token under My Account > Security in SonarQube.", + ) + case !strings.HasPrefix(token, userTokenPrefix): + return errors.BadInput.New( + "DevLake requires a User token (squ_ prefix) for SonarQube Server. " + + "Create one under My Account > Security in SonarQube.", + ) + } + return nil +} + +func (connection SonarqubeConn) IsCloud() bool { return connection.Endpoint == "https://sonarcloud.io/api/" } diff --git a/backend/plugins/sonarqube/models/connection_test.go b/backend/plugins/sonarqube/models/connection_test.go new file mode 100644 index 00000000000..e92284ad5da --- /dev/null +++ b/backend/plugins/sonarqube/models/connection_test.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "testing" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func TestValidateUserTokenPrefix(t *testing.T) { + t.Parallel() + + serverEndpoint := "https://rad-sonar.example.com/api/" + cloudEndpoint := "https://sonarcloud.io/api/" + + tests := []struct { + name string + conn SonarqubeConn + wantErr bool + }{ + { + name: "user token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "squ_abc123"}, + }, + }, + { + name: "global analysis token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"}, + }, + wantErr: true, + }, + { + name: "project analysis token on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqp_abc123"}, + }, + wantErr: true, + }, + { + name: "unknown prefix on server", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: serverEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "legacy-token"}, + }, + wantErr: true, + }, + { + name: "sonarcloud skips prefix check", + conn: SonarqubeConn{ + RestConnection: helper.RestConnection{Endpoint: cloudEndpoint}, + SonarqubeAccessToken: SonarqubeAccessToken{Token: "sqa_abc123"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + err := tt.conn.ValidateUserTokenPrefix() + if tt.wantErr { + if err == nil { + t.Fatal("expected error, got nil") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + } +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go b/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go new file mode 100644 index 00000000000..3ba3a00eb2e --- /dev/null +++ b/backend/plugins/sonarqube/models/migrationscripts/20260701_change_issue_code_block_component_type.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.MigrationScript = (*changeIssueCodeBlockComponentType)(nil) + +type changeIssueCodeBlockComponentType struct{} + +func (script *changeIssueCodeBlockComponentType) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + if err := db.DropIndexes("_tool_sonarqube_issue_code_blocks", "idx__tool_sonarqube_issue_code_blocks_component"); err != nil { + return err + } + return db.ModifyColumnType("_tool_sonarqube_issue_code_blocks", "component", "text") +} + +func (*changeIssueCodeBlockComponentType) Version() uint64 { + return 20260701000000 +} + +func (*changeIssueCodeBlockComponentType) Name() string { + return "change _tool_sonarqube_issue_code_blocks.component type to text" +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go b/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go new file mode 100644 index 00000000000..d060f46310a --- /dev/null +++ b/backend/plugins/sonarqube/models/migrationscripts/20260707_add_project_metrics_history.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +var _ plugin.MigrationScript = (*addProjectMetricsHistory)(nil) + +type projectMetricsHistory20260707 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"primaryKey"` + MetricKey string `gorm:"primaryKey;type:varchar(100)"` + MetricValue string `gorm:"type:varchar(50)"` + archived.NoPKModel +} + +func (projectMetricsHistory20260707) TableName() string { + return "_tool_sonarqube_project_metrics_history" +} + +type projectAnalyses20260707 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"index"` + ProjectVersion string `gorm:"type:varchar(255)"` + Revision string `gorm:"type:varchar(255)"` + BuildString string `gorm:"type:varchar(255)"` + DetectedCI string `gorm:"type:varchar(100)"` + archived.NoPKModel +} + +func (projectAnalyses20260707) TableName() string { + return "_tool_sonarqube_project_analyses" +} + +type addProjectMetricsHistory struct{} + +func (script *addProjectMetricsHistory) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &projectMetricsHistory20260707{}, + &projectAnalyses20260707{}, + ) +} + +func (*addProjectMetricsHistory) Version() uint64 { + return 20260707153200 +} + +func (*addProjectMetricsHistory) Name() string { + return "add project_metrics_history and project_analyses tables" +} diff --git a/backend/plugins/sonarqube/models/migrationscripts/register.go b/backend/plugins/sonarqube/models/migrationscripts/register.go index 7c48de84226..4e1e590b9fa 100644 --- a/backend/plugins/sonarqube/models/migrationscripts/register.go +++ b/backend/plugins/sonarqube/models/migrationscripts/register.go @@ -39,5 +39,7 @@ func All() []plugin.MigrationScript { new(addOrgToConn), new(addIssueImpacts), new(extendSonarqubeFieldSize), + new(changeIssueCodeBlockComponentType), + new(addProjectMetricsHistory), } } diff --git a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go index b5bb58bddae..b36386338da 100644 --- a/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go +++ b/backend/plugins/sonarqube/models/sonarqube_issue_code_block.go @@ -23,7 +23,7 @@ type SonarqubeIssueCodeBlock struct { ConnectionId uint64 `gorm:"primaryKey"` Id string `gorm:"primaryKey"` IssueKey string `gorm:"index"` - Component string `gorm:"index;type:varchar(500)"` + Component string `gorm:"type:text"` StartLine int EndLine int StartOffset int diff --git a/backend/plugins/sonarqube/models/sonarqube_project_analyses.go b/backend/plugins/sonarqube/models/sonarqube_project_analyses.go new file mode 100644 index 00000000000..55dac5ff702 --- /dev/null +++ b/backend/plugins/sonarqube/models/sonarqube_project_analyses.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type SonarqubeProjectAnalysis struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"index"` + ProjectVersion string `gorm:"type:varchar(255)"` + Revision string `gorm:"type:varchar(255)"` + BuildString string `gorm:"type:varchar(255)"` + DetectedCI string `gorm:"type:varchar(100)"` + common.NoPKModel +} + +func (SonarqubeProjectAnalysis) TableName() string { + return "_tool_sonarqube_project_analyses" +} diff --git a/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go b/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go new file mode 100644 index 00000000000..ebd555252ac --- /dev/null +++ b/backend/plugins/sonarqube/models/sonarqube_project_metrics_history.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +type SonarqubeProjectMetricsHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectKey string `gorm:"primaryKey;type:varchar(255)"` + AnalysisDate time.Time `gorm:"primaryKey"` + MetricKey string `gorm:"primaryKey;type:varchar(100)"` + MetricValue string `gorm:"type:varchar(50)"` + common.NoPKModel +} + +func (SonarqubeProjectMetricsHistory) TableName() string { + return "_tool_sonarqube_project_metrics_history" +} diff --git a/backend/plugins/sonarqube/tasks/accounts_collector.go b/backend/plugins/sonarqube/tasks/accounts_collector.go index 6bb02272e12..0a30b57f014 100644 --- a/backend/plugins/sonarqube/tasks/accounts_collector.go +++ b/backend/plugins/sonarqube/tasks/accounts_collector.go @@ -36,11 +36,20 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { logger := taskCtx.GetLogger() logger.Info("collect accounts") rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_ACCOUNTS_TABLE) + + // SonarCloud removed api/users/search - use organizations/search_members instead + // The "organization" query param is auto-added by PrepareApiClient + urlTemplate := "users/search" + if data.IsCloud { + urlTemplate = "organizations/search_members" + logger.Info("using organizations/search_members for SonarCloud") + } + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, PageSize: 100, - UrlTemplate: "users/search", + UrlTemplate: urlTemplate, Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { query := url.Values{} query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page)) diff --git a/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go new file mode 100644 index 00000000000..c3d703fe2a0 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/issue_code_blocks_convertor_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" + "github.com/stretchr/testify/require" +) + +func TestIssueCodeBlockComponentIsText(t *testing.T) { + testCases := []struct { + name string + model interface{} + }{ + {name: "tool layer", model: models.SonarqubeIssueCodeBlock{}}, + {name: "domain layer", model: codequality.CqIssueCodeBlock{}}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + field, ok := reflect.TypeOf(testCase.model).FieldByName("Component") + require.True(t, ok) + require.Equal(t, "type:text", field.Tag.Get("gorm")) + }) + } +} diff --git a/backend/plugins/sonarqube/tasks/project_analyses_collector.go b/backend/plugins/sonarqube/tasks/project_analyses_collector.go new file mode 100644 index 00000000000..37682fd7cd0 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_analyses_collector.go @@ -0,0 +1,92 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +const RAW_PROJECT_ANALYSES_TABLE = "sonarqube_api_project_analyses" + +var _ plugin.SubTaskEntryPoint = CollectProjectAnalyses + +func CollectProjectAnalyses(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + logger.Info("collect project analyses") + + data := taskCtx.GetData().(*SonarqubeTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.SonarqubeApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + }, + Table: RAW_PROJECT_ANALYSES_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: 500, + UrlTemplate: "project_analyses/search", + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", data.Options.ProjectKey) + query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page)) + if apiCollector.GetSince() != nil { + query.Set("from", apiCollector.GetSince().UTC().Format("2006-01-02")) + } + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var resData struct { + Analyses []json.RawMessage `json:"analyses"` + } + err := helper.UnmarshalResponse(res, &resData) + if err != nil { + return nil, err + } + return resData.Analyses, nil + }, + }) + if err != nil { + return err + } + + return apiCollector.Execute() +} + +var CollectProjectAnalysesMeta = plugin.SubTaskMeta{ + Name: "CollectProjectAnalyses", + EntryPoint: CollectProjectAnalyses, + EnabledByDefault: true, + Description: "Collect project analysis metadata from SonarQube project_analyses/search API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_analyses_extractor.go b/backend/plugins/sonarqube/tasks/project_analyses_extractor.go new file mode 100644 index 00000000000..ac10d8d6e09 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_analyses_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProjectAnalyses + +type projectAnalysisResponse struct { + Key string `json:"key"` + Date string `json:"date"` + ProjectVersion string `json:"projectVersion"` + Revision string `json:"revision"` + BuildString string `json:"buildString"` + DetectedCI string `json:"detectedCI"` +} + +func ExtractProjectAnalyses(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_ANALYSES_TABLE) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) { + body := &projectAnalysisResponse{} + err := errors.Convert(json.Unmarshal(resData.Data, body)) + if err != nil { + return nil, err + } + + analysisDate, parseErr := time.Parse("2006-01-02T15:04:05-0700", body.Date) + if parseErr != nil { + return nil, errors.Default.Wrap(errors.Convert(parseErr), "failed to parse analysis date") + } + + analysis := &models.SonarqubeProjectAnalysis{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + AnalysisKey: body.Key, + AnalysisDate: analysisDate, + ProjectVersion: body.ProjectVersion, + Revision: body.Revision, + BuildString: body.BuildString, + DetectedCI: body.DetectedCI, + } + return []interface{}{analysis}, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} + +var ExtractProjectAnalysesMeta = plugin.SubTaskMeta{ + Name: "ExtractProjectAnalyses", + EntryPoint: ExtractProjectAnalyses, + EnabledByDefault: true, + Description: "Extract raw project analyses data into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go b/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go new file mode 100644 index 00000000000..fa2b02633b6 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_collector.go @@ -0,0 +1,94 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +const RAW_PROJECT_METRICS_HISTORY_TABLE = "sonarqube_api_project_metrics_history" + +const metricsToCollect = "coverage,ncloc,bugs,vulnerabilities,code_smells,security_hotspots," + + "duplicated_lines_density,sqale_rating,reliability_rating,security_rating,complexity,cognitive_complexity" + +var _ plugin.SubTaskEntryPoint = CollectProjectMetricsHistory + +func CollectProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + logger.Info("collect project metrics history") + + data := taskCtx.GetData().(*SonarqubeTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.SonarqubeApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + }, + Table: RAW_PROJECT_METRICS_HISTORY_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: data.ApiClient, + PageSize: 1000, + UrlTemplate: "measures/search_history", + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("component", data.Options.ProjectKey) + query.Set("metrics", metricsToCollect) + query.Set("ps", fmt.Sprintf("%v", reqData.Pager.Size)) + query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page)) + if apiCollector.GetSince() != nil { + query.Set("from", apiCollector.GetSince().UTC().Format("2006-01-02")) + } + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var body json.RawMessage + err := helper.UnmarshalResponse(res, &body) + if err != nil { + return nil, err + } + return []json.RawMessage{body}, nil + }, + }) + if err != nil { + return err + } + + return apiCollector.Execute() +} + +var CollectProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "CollectProjectMetricsHistory", + EntryPoint: CollectProjectMetricsHistory, + EnabledByDefault: true, + Description: "Collect project-level metric history from SonarQube measures/search_history API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go b/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go new file mode 100644 index 00000000000..bf59aa1fe98 --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_convertor.go @@ -0,0 +1,158 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/codequality" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + sonarqubeModels "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var ConvertProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "convertProjectMetricsHistory", + EntryPoint: ConvertProjectMetricsHistory, + EnabledByDefault: true, + Description: "Convert tool layer project metrics history into domain layer table cq_project_metrics_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} + +func ConvertProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + _, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_METRICS_HISTORY_TABLE) + + // Query all narrow metric rows ordered so we can group-pivot them + cursor, err := db.Cursor( + dal.From(sonarqubeModels.SonarqubeProjectMetricsHistory{}), + dal.Where("connection_id = ? AND project_key = ?", data.Options.ConnectionId, data.Options.ProjectKey), + dal.Orderby("analysis_date"), + ) + if err != nil { + return err + } + defer cursor.Close() + + projectIdGen := didgen.NewDomainIdGenerator(&sonarqubeModels.SonarqubeProject{}) + domainProjectKey := projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectKey) + + batchSave, err := helper.NewBatchSave(taskCtx, reflect.TypeOf(&codequality.CqProjectMetricsHistory{}), 200) + if err != nil { + return err + } + defer batchSave.Close() + + // Group narrow rows by analysis_date, pivot into wide domain rows + var currentDate *time.Time + var currentDomain *codequality.CqProjectMetricsHistory + + flushCurrent := func() errors.Error { + if currentDomain != nil { + return batchSave.Add(currentDomain) + } + return nil + } + + for cursor.Next() { + row := &sonarqubeModels.SonarqubeProjectMetricsHistory{} + err = db.Fetch(cursor, row) + if err != nil { + return err + } + + if currentDate == nil || !currentDate.Equal(row.AnalysisDate) { + if flushErr := flushCurrent(); flushErr != nil { + return flushErr + } + domainId := fmt.Sprintf("%s:%s", + domainProjectKey, + row.AnalysisDate.UTC().Format("2006-01-02T15:04:05Z"), + ) + currentDomain = &codequality.CqProjectMetricsHistory{ + DomainEntity: domainlayer.DomainEntity{Id: domainId}, + ProjectKey: domainProjectKey, + AnalysisDate: row.AnalysisDate, + } + t := row.AnalysisDate + currentDate = &t + } + + applyMetricValue(currentDomain, row.MetricKey, row.MetricValue) + } + + if flushErr := flushCurrent(); flushErr != nil { + return flushErr + } + + return batchSave.Close() +} + +func applyMetricValue(d *codequality.CqProjectMetricsHistory, metricKey, value string) { + switch metricKey { + case "coverage": + if v, err := strconv.ParseFloat(value, 64); err == nil { + d.Coverage = &v + } + case "ncloc": + if v, err := strconv.Atoi(value); err == nil { + d.Ncloc = &v + } + case "bugs": + if v, err := strconv.Atoi(value); err == nil { + d.Bugs = &v + } + case "reliability_rating": + d.ReliabilityRating = alphabetMap[value] + case "code_smells": + if v, err := strconv.Atoi(value); err == nil { + d.CodeSmells = &v + } + case "sqale_rating": + d.SqaleRating = alphabetMap[value] + case "complexity": + if v, err := strconv.Atoi(value); err == nil { + d.Complexity = &v + } + case "cognitive_complexity": + if v, err := strconv.Atoi(value); err == nil { + d.CognitiveComplexity = &v + } + case "vulnerabilities": + if v, err := strconv.Atoi(value); err == nil { + d.Vulnerabilities = &v + } + case "security_rating": + d.SecurityRating = alphabetMap[value] + case "security_hotspots": + if v, err := strconv.Atoi(value); err == nil { + d.SecurityHotspots = &v + } + case "duplicated_lines_density": + if v, err := strconv.ParseFloat(value, 64); err == nil { + d.DuplicatedLinesDensity = &v + } + } +} diff --git a/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go b/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go new file mode 100644 index 00000000000..430ff196beb --- /dev/null +++ b/backend/plugins/sonarqube/tasks/project_metrics_history_extractor.go @@ -0,0 +1,89 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/sonarqube/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProjectMetricsHistory + +type metricsHistoryResponse struct { + Measures []struct { + Metric string `json:"metric"` + History []struct { + Date string `json:"date"` + Value string `json:"value"` + } `json:"history"` + } `json:"measures"` +} + +func ExtractProjectMetricsHistory(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_PROJECT_METRICS_HISTORY_TABLE) + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(resData *helper.RawData) ([]interface{}, errors.Error) { + body := &metricsHistoryResponse{} + err := errors.Convert(json.Unmarshal(resData.Data, body)) + if err != nil { + return nil, err + } + + var results []interface{} + for _, measure := range body.Measures { + for _, entry := range measure.History { + if entry.Value == "" { + continue + } + analysisDate, parseErr := time.Parse("2006-01-02T15:04:05-0700", entry.Date) + if parseErr != nil { + return nil, errors.Default.Wrap(errors.Convert(parseErr), "failed to parse analysis date") + } + results = append(results, &models.SonarqubeProjectMetricsHistory{ + ConnectionId: data.Options.ConnectionId, + ProjectKey: data.Options.ProjectKey, + AnalysisDate: analysisDate, + MetricKey: measure.Metric, + MetricValue: entry.Value, + }) + } + } + return results, nil + }, + }) + if err != nil { + return err + } + + return extractor.Execute() +} + +var ExtractProjectMetricsHistoryMeta = plugin.SubTaskMeta{ + Name: "ExtractProjectMetricsHistory", + EntryPoint: ExtractProjectMetricsHistory, + EnabledByDefault: true, + Description: "Extract raw project metrics history into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_QUALITY}, +} diff --git a/backend/plugins/sonarqube/tasks/task_data.go b/backend/plugins/sonarqube/tasks/task_data.go index e898befa0db..06cb83dd5b0 100644 --- a/backend/plugins/sonarqube/tasks/task_data.go +++ b/backend/plugins/sonarqube/tasks/task_data.go @@ -37,6 +37,7 @@ type SonarqubeTaskData struct { ApiClient *api.ApiAsyncClient LastAnalysisDate *time.Time TaskStartTime time.Time + IsCloud bool } func DecodeAndValidateTaskOptions(options map[string]interface{}) (*SonarqubeOptions, errors.Error) { diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index ce0baca4974..b05667c7416 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -25,6 +25,7 @@ import ( aidetector "github.com/apache/incubator-devlake/plugins/aidetector/impl" aimeasure "github.com/apache/incubator-devlake/plugins/aimeasure/impl" argocd "github.com/apache/incubator-devlake/plugins/argocd/impl" + asana "github.com/apache/incubator-devlake/plugins/asana/impl" azuredevops "github.com/apache/incubator-devlake/plugins/azuredevops_go/impl" bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" @@ -33,11 +34,13 @@ import ( capacityplanner "github.com/apache/incubator-devlake/plugins/capacityplanner/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" cursor "github.com/apache/incubator-devlake/plugins/cursor/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" feishu "github.com/apache/incubator-devlake/plugins/feishu/impl" findevops "github.com/apache/incubator-devlake/plugins/findevops/impl" + copilot "github.com/apache/incubator-devlake/plugins/gh-copilot/impl" gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" github "github.com/apache/incubator-devlake/plugins/github/impl" @@ -47,17 +50,21 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" q_dev "github.com/apache/incubator-devlake/plugins/q_dev/impl" refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" + rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" slack "github.com/apache/incubator-devlake/plugins/slack/impl" sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl" starrocks "github.com/apache/incubator-devlake/plugins/starrocks/impl" + taiga "github.com/apache/incubator-devlake/plugins/taiga/impl" tapd "github.com/apache/incubator-devlake/plugins/tapd/impl" teambition "github.com/apache/incubator-devlake/plugins/teambition/impl" + tempo "github.com/apache/incubator-devlake/plugins/tempo/impl" testmo "github.com/apache/incubator-devlake/plugins/testmo/impl" trello "github.com/apache/incubator-devlake/plugins/trello/impl" webhook "github.com/apache/incubator-devlake/plugins/webhook/impl" @@ -79,6 +86,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("capacityplanner/models", capacityplanner.CapacityPlanner{}.GetTablesInfo) checker.FeedIn("bitbucket_server/models", bitbucket_server.BitbucketServer{}.GetTablesInfo) checker.FeedIn("argocd/models", argocd.ArgoCD{}.GetTablesInfo) + checker.FeedIn("asana/models", asana.Asana{}.GetTablesInfo) checker.FeedIn("customize/models", customize.Customize{}.GetTablesInfo) checker.FeedIn("dbt", dbt.Dbt{}.GetTablesInfo) checker.FeedIn("dora/models", dora.Dora{}.GetTablesInfo) @@ -92,24 +100,30 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) checker.FeedIn("jenkins/models", jenkins.Jenkins{}.GetTablesInfo) checker.FeedIn("jira/models", jira.Jira{}.GetTablesInfo) + checker.FeedIn("linear/models", linear.Linear{}.GetTablesInfo) checker.FeedIn("org", org.Org{}.GetTablesInfo) checker.FeedIn("pagerduty/models", pagerduty.PagerDuty{}.GetTablesInfo) checker.FeedIn("refdiff/models", refdiff.RefDiff{}.GetTablesInfo) + checker.FeedIn("rootly/models", rootly.Rootly{}.GetTablesInfo) checker.FeedIn("slack/models", slack.Slack{}.GetTablesInfo) checker.FeedIn("sonarqube/models", sonarqube.Sonarqube{}.GetTablesInfo) checker.FeedIn("starrocks", starrocks.StarRocks{}.GetTablesInfo) + checker.FeedIn("taiga/models", taiga.Taiga{}.GetTablesInfo) checker.FeedIn("tapd/models", tapd.Tapd{}.GetTablesInfo) checker.FeedIn("teambition/models", teambition.Teambition{}.GetTablesInfo) + checker.FeedIn("tempo/models", tempo.Tempo{}.GetTablesInfo) checker.FeedIn("testmo/models", testmo.Testmo{}.GetTablesInfo) checker.FeedIn("trello/models", trello.Trello{}.GetTablesInfo) checker.FeedIn("webhook/models", webhook.Webhook{}.GetTablesInfo) checker.FeedIn("zentao/models", zentao.Zentao{}.GetTablesInfo) + checker.FeedIn("claude_code/models", claudeCode.ClaudeCode{}.GetTablesInfo) checker.FeedIn("circleci/models", circleci.Circleci{}.GetTablesInfo) checker.FeedIn("opsgenie/models", opsgenie.Opsgenie{}.GetTablesInfo) checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) checker.FeedIn("cursor/models", cursor.Cursor{}.GetTablesInfo) + checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) err := checker.Verify() if err != nil { t.Error(err) diff --git a/backend/plugins/taiga/api/blueprint_v200.go b/backend/plugins/taiga/api/blueprint_v200.go new file mode 100644 index 00000000000..3eabf1acbd5 --- /dev/null +++ b/backend/plugins/taiga/api/blueprint_v200.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +type TaigaTaskOptions struct { + ConnectionId uint64 `json:"connectionId"` + ProjectId uint64 `json:"projectId"` +} + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + // load connection, scope and scopeConfig from the db + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + // needed for the connection to populate its access tokens + _, err = helper.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + return plan, scopes, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.TaigaProject, models.TaigaScopeConfig], + connection *models.TaigaConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + // construct task options for Taiga + task, err := helper.MakePipelinePlanTask( + "taiga", + subtaskMetas, + scopeConfig.Entities, + TaigaTaskOptions{ + ConnectionId: scope.ConnectionId, + ProjectId: uint64(scope.ProjectId), + }, + ) + if err != nil { + return nil, err + } + + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.TaigaProject, models.TaigaScopeConfig], + connection *models.TaigaConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + + for _, scopeDetail := range scopeDetails { + project := scopeDetail.Scope + + // add board to scopes + entities := scopeDetail.ScopeConfig.Entities + hasTicket := false + for _, entity := range entities { + if entity == plugin.DOMAIN_TYPE_TICKET { + hasTicket = true + break + } + } + if hasTicket { + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: idGen.Generate(connection.ID, project.ProjectId), + }, + Name: project.Name, + } + scopes = append(scopes, domainBoard) + } + } + + return scopes, nil +} diff --git a/backend/plugins/taiga/api/connection_api.go b/backend/plugins/taiga/api/connection_api.go new file mode 100644 index 00000000000..88021be3a99 --- /dev/null +++ b/backend/plugins/taiga/api/connection_api.go @@ -0,0 +1,225 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "fmt" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +// TaigaTestConnResponse is the response struct for testing a connection +type TaigaTestConnResponse struct { + shared.ApiBody + Connection *models.TaigaConnection +} + +// testConnection tests the Taiga connection +func testConnection(ctx context.Context, connection models.TaigaConnection) (*TaigaTestConnResponse, errors.Error) { + // If username and password are provided, authenticate to get a token + if connection.Username != "" && connection.Password != "" && connection.Token == "" { + // Create a temporary connection without token for authentication + tempConnection := connection + tempConnection.Token = "" + + // Create a temporary API client to call the auth endpoint + tempApiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &tempConnection) + if err != nil { + return nil, errors.Default.Wrap(err, "error creating API client") + } + + // Prepare auth request body + authBody := map[string]interface{}{ + "type": "normal", + "username": connection.Username, + "password": connection.Password, + } + + // Authenticate to get token + authResponse := struct { + AuthToken string `json:"auth_token"` + }{} + + res, err := tempApiClient.Post("auth", nil, authBody, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "error authenticating with Taiga") + } + + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusBadRequest { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed - please check your username and password") + } + + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code during auth: %d", res.StatusCode)) + } + + // Parse the auth response + err = api.UnmarshalResponse(res, &authResponse) + if err != nil { + return nil, errors.Default.Wrap(err, "error parsing authentication response") + } + + // Set the token for validation + connection.Token = authResponse.AuthToken + } + + // validate - but make Token optional if we have username/password + if vld != nil { + if connection.Token == "" && (connection.Username == "" || connection.Password == "") { + return nil, errors.Default.New("either token or username/password must be provided") + } + } + + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + + // test connection by making a request to the user endpoint + res, err := apiClient.Get("users/me", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "error testing connection") + } + + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication error when testing connection - please check your credentials") + } + + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code: %d", res.StatusCode)) + } + + connection = connection.Sanitize() + body := TaigaTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + + return &body, nil +} + +// TestConnection tests the Taiga connection +// @Summary test taiga connection +// @Description Test Taiga Connection +// @Tags plugins/taiga +// @Param body body models.TaigaConnection true "json body" +// @Success 200 {object} TaigaTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/taiga/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // decode + var connection models.TaigaConnection + err := api.DecodeMapStruct(input.Body, &connection, false) + if err != nil { + return nil, err + } + // test connection + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection tests an existing Taiga connection +// @Summary test existing taiga connection +// @Description Test Existing Taiga Connection +// @Tags plugins/taiga +// @Success 200 {object} TaigaTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/taiga/connections/:connectionId/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + // test connection + result, err := testConnection(context.TODO(), *connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections creates a new Taiga connection +// @Summary create taiga connection +// @Description Create Taiga Connection +// @Tags plugins/taiga +// @Success 200 {object} models.TaigaConnection +// @Failure 400 +// @Failure 500 +// @Router /plugins/taiga/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// ListConnections lists all Taiga connections +// @Summary list taiga connections +// @Description List Taiga Connections +// @Tags plugins/taiga +// @Success 200 {object} []models.TaigaConnection +// @Failure 400 +// @Failure 500 +// @Router /plugins/taiga/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection gets a Taiga connection by ID +// @Summary get taiga connection +// @Description Get Taiga Connection +// @Tags plugins/taiga +// @Success 200 {object} models.TaigaConnection +// @Failure 400 +// @Failure 500 +// @Router /plugins/taiga/connections/:connectionId [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} + +// PatchConnection updates a Taiga connection +// @Summary patch taiga connection +// @Description Patch Taiga Connection +// @Tags plugins/taiga +// @Success 200 {object} models.TaigaConnection +// @Failure 400 +// @Failure 500 +// @Router /plugins/taiga/connections/:connectionId [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection deletes a Taiga connection +// @Summary delete taiga connection +// @Description Delete Taiga Connection +// @Tags plugins/taiga +// @Success 200 +// @Failure 400 +// @Failure 500 +// @Router /plugins/taiga/connections/:connectionId [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} diff --git a/backend/plugins/taiga/api/init.go b/backend/plugins/taiga/api/init.go new file mode 100644 index 00000000000..82b332456ac --- /dev/null +++ b/backend/plugins/taiga/api/init.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.TaigaConnection, models.TaigaProject, models.TaigaScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.TaigaConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.TaigaConnection, models.TaigaProject, TaigaRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.TaigaConnection, + models.TaigaProject, + models.TaigaScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.TaigaConnection) models.TaigaConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.TaigaConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.TaigaConnection, models.TaigaProject, TaigaRemotePagination](raProxy, listTaigaRemoteScopes) +} diff --git a/backend/plugins/taiga/api/remote_api.go b/backend/plugins/taiga/api/remote_api.go new file mode 100644 index 00000000000..a0cc0b02a90 --- /dev/null +++ b/backend/plugins/taiga/api/remote_api.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +type TaigaRemotePagination struct { + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +type TaigaApiProject struct { + Id uint64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` + Description string `json:"description"` +} + +func queryTaigaProjects( + apiClient plugin.ApiClient, + keyword string, + page TaigaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TaigaProject], + nextPage *TaigaRemotePagination, + err errors.Error, +) { + if page.PageSize == 0 { + page.PageSize = 100 + } + if page.Page == 0 { + page.Page = 1 + } + + query := url.Values{ + "page": {fmt.Sprintf("%d", page.Page)}, + "page_size": {fmt.Sprintf("%d", page.PageSize)}, + } + if keyword != "" { + query.Set("search", keyword) + } + + res, err := apiClient.Get("projects", query, nil) + if err != nil { + return + } + + var projects []TaigaApiProject + err = api.UnmarshalResponse(res, &projects) + if err != nil { + return + } + + for _, project := range projects { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TaigaProject]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: fmt.Sprintf("%d", project.Id), + ParentId: nil, + Name: project.Name, + FullName: project.Name, + Data: &models.TaigaProject{ + ProjectId: project.Id, + Name: project.Name, + Slug: project.Slug, + Description: project.Description, + }, + }) + } + + // Check if there are more pages + if len(projects) == page.PageSize { + nextPage = &TaigaRemotePagination{ + Page: page.Page + 1, + PageSize: page.PageSize, + } + } + + return +} + +func listTaigaRemoteScopes( + _ *models.TaigaConnection, + apiClient plugin.ApiClient, + groupId string, + page TaigaRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TaigaProject], + nextPage *TaigaRemotePagination, + err errors.Error, +) { + return queryTaigaProjects(apiClient, "", page) +} + +// RemoteScopes list all available scopes on the remote server +// @Summary list all available scopes on the remote server +// @Description list all available scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TaigaProject] +// @Tags plugins/taiga +// @Router /plugins/taiga/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} diff --git a/backend/plugins/taiga/api/scope_api.go b/backend/plugins/taiga/api/scope_api.go new file mode 100644 index 00000000000..3b8c128894d --- /dev/null +++ b/backend/plugins/taiga/api/scope_api.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.TaigaProject] +type ScopeDetail api.ScopeDetail[models.TaigaProject, models.TaigaScopeConfig] + +func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/taiga/api/scope_config_api.go b/backend/plugins/taiga/api/scope_config_api.go new file mode 100644 index 00000000000..f162da31c07 --- /dev/null +++ b/backend/plugins/taiga/api/scope_config_api.go @@ -0,0 +1,47 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/taiga/e2e/epic_test.go b/backend/plugins/taiga/e2e/epic_test.go new file mode 100644 index 00000000000..47ac0c5d962 --- /dev/null +++ b/backend/plugins/taiga/e2e/epic_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +// TestTaigaEpicDataFlow verifies the full extract → convert pipeline for +// Taiga epics: raw API JSON → _tool_taiga_epics → ticket.Issue (type EPIC). +// +// To regenerate snapshot CSVs from actual output, temporarily replace +// VerifyTableWithOptions with CreateSnapshot for each table. +func TestTaigaEpicDataFlow(t *testing.T) { + var taiga impl.Taiga + dataflowTester := e2ehelper.NewDataFlowTester(t, "taiga", taiga) + + taskData := &tasks.TaigaTaskData{ + Options: &tasks.TaigaOptions{ + ConnectionId: 1, + ProjectId: 1, + }, + } + + // ── Extraction: raw JSON → _tool_taiga_epics ───────────────────────────── + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_taiga_api_epics.csv", + "_raw_taiga_api_epics", + ) + + dataflowTester.FlushTabler(&models.TaigaEpic{}) + dataflowTester.Subtask(tasks.ExtractEpicsMeta, taskData) + // Verify all tool-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(models.TaigaEpic{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_taiga_epics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // ── Conversion: _tool_taiga_epics → ticket.Issue / ticket.BoardIssue ───── + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertEpicsMeta, taskData) + // Verify all domain-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_from_epics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues_from_epics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/taiga/e2e/issue_test.go b/backend/plugins/taiga/e2e/issue_test.go new file mode 100644 index 00000000000..7ab3c9c85a9 --- /dev/null +++ b/backend/plugins/taiga/e2e/issue_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +// TestTaigaIssueDataFlow verifies the full extract → convert pipeline for +// Taiga issues (bugs/enhancements/questions): raw API JSON → +// _tool_taiga_issues → ticket.Issue with correct DevLake type mapping. +// Specifically checks that "Bug" → BUG, "Enhancement" → REQUIREMENT. +// +// To regenerate snapshot CSVs from actual output, temporarily replace +// VerifyTableWithOptions with CreateSnapshot for each table. +func TestTaigaIssueDataFlow(t *testing.T) { + var taiga impl.Taiga + dataflowTester := e2ehelper.NewDataFlowTester(t, "taiga", taiga) + + taskData := &tasks.TaigaTaskData{ + Options: &tasks.TaigaOptions{ + ConnectionId: 1, + ProjectId: 1, + }, + } + + // ── Extraction: raw JSON → _tool_taiga_issues ──────────────────────────── + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_taiga_api_issues.csv", + "_raw_taiga_api_issues", + ) + + dataflowTester.FlushTabler(&models.TaigaIssue{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + // Verify all tool-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(models.TaigaIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_taiga_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // ── Conversion: _tool_taiga_issues → ticket.Issue / ticket.BoardIssue ──── + // Key assertion: taigaIssueTypeToDevLake mapping is exercised (Bug→BUG, Enhancement→REQUIREMENT). + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + // Verify all domain-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_from_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues_from_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/taiga/e2e/project_test.go b/backend/plugins/taiga/e2e/project_test.go new file mode 100644 index 00000000000..d1e8aa87096 --- /dev/null +++ b/backend/plugins/taiga/e2e/project_test.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +// TestTaigaProjectDataFlow verifies the full extract → convert pipeline for +// Taiga projects: raw API JSON → _tool_taiga_projects → ticket.Board. +// +// To regenerate snapshot CSVs from actual output, temporarily replace +// VerifyTableWithOptions with CreateSnapshot for each table. +func TestTaigaProjectDataFlow(t *testing.T) { + var taiga impl.Taiga + dataflowTester := e2ehelper.NewDataFlowTester(t, "taiga", taiga) + + taskData := &tasks.TaigaTaskData{ + Options: &tasks.TaigaOptions{ + ConnectionId: 1, + ProjectId: 1, + }, + } + + // ── Extraction: raw JSON → _tool_taiga_projects ───────────────────────── + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_taiga_api_projects.csv", + "_raw_taiga_api_projects", + ) + + dataflowTester.FlushTabler(&models.TaigaProject{}) + dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData) + // Verify all tool-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(models.TaigaProject{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_taiga_projects.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // ── Conversion: _tool_taiga_projects → ticket.Board ────────────────────── + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertProjectsMeta, taskData) + // Verify all domain-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_epics.csv b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_epics.csv new file mode 100644 index 00000000000..9121a54d0a3 --- /dev/null +++ b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_epics.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":401,""ref"":1,""subject"":""Platform v2.0"",""status"":1,""status_extra_info"":{""name"":""In Progress""},""is_closed"":false,""created_date"":""2024-01-05T09:00:00.000Z"",""modified_date"":""2024-01-20T10:00:00.000Z"",""assigned_to"":5,""assigned_to_extra_info"":{""full_name_display"":""Alice Smith""},""color"":""#F2711C""}",https://api.taiga.io/api/v1/epics?project=1,null,2024-01-21 08:00:00.000 +2,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":402,""ref"":2,""subject"":""Mobile App"",""status"":2,""status_extra_info"":{""name"":""Open""},""is_closed"":false,""created_date"":""2024-01-07T11:00:00.000Z"",""modified_date"":""2024-01-18T09:00:00.000Z"",""assigned_to"":null,""assigned_to_extra_info"":null,""color"":""#2185D0""}",https://api.taiga.io/api/v1/epics?project=1,null,2024-01-21 08:00:00.000 diff --git a/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_issues.csv b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_issues.csv new file mode 100644 index 00000000000..5e56af72500 --- /dev/null +++ b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_issues.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":301,""ref"":1,""subject"":""Login page crash on mobile"",""status"":1,""status_extra_info"":{""name"":""New""},""type_extra_info"":{""name"":""Bug""},""priority_extra_info"":{""name"":""High""},""severity_extra_info"":{""name"":""Critical""},""is_closed"":false,""created_date"":""2024-01-14T08:00:00.000Z"",""modified_date"":""2024-01-19T14:00:00.000Z"",""finished_date"":null,""assigned_to"":5,""assigned_to_extra_info"":{""full_name_display"":""Alice Smith""},""milestone"":null}",https://api.taiga.io/api/v1/issues?project=1,null,2024-01-21 08:00:00.000 +2,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":302,""ref"":2,""subject"":""Add dark mode support"",""status"":2,""status_extra_info"":{""name"":""Closed""},""type_extra_info"":{""name"":""Enhancement""},""priority_extra_info"":{""name"":""Normal""},""severity_extra_info"":{""name"":""Minor""},""is_closed"":true,""created_date"":""2024-01-08T10:00:00.000Z"",""modified_date"":""2024-01-16T13:00:00.000Z"",""finished_date"":""2024-01-16T13:00:00.000Z"",""assigned_to"":6,""assigned_to_extra_info"":{""full_name_display"":""Bob Jones""},""milestone"":1}",https://api.taiga.io/api/v1/issues?project=1,null,2024-01-21 08:00:00.000 diff --git a/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_projects.csv b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_projects.csv new file mode 100644 index 00000000000..2871848b092 --- /dev/null +++ b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_projects.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":1,""name"":""Test Project Alpha"",""slug"":""test-project-alpha"",""description"":""A test project for the DevLake Taiga plugin e2e tests"",""created_date"":""2024-01-01T00:00:00.000Z"",""modified_date"":""2024-01-20T00:00:00.000Z""}",https://api.taiga.io/api/v1/projects/1,null,2024-01-21 08:00:00.000 diff --git a/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_tasks.csv b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_tasks.csv new file mode 100644 index 00000000000..227fff104a2 --- /dev/null +++ b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_tasks.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":201,""ref"":1,""subject"":""Write unit tests"",""status"":1,""status_extra_info"":{""name"":""In Progress""},""is_closed"":false,""created_date"":""2024-01-16T09:00:00.000Z"",""modified_date"":""2024-01-21T11:00:00.000Z"",""finished_date"":null,""assigned_to"":5,""assigned_to_extra_info"":{""full_name_display"":""Alice Smith""},""user_story"":101,""milestone"":1,""is_blocked"":false,""blocked_note"":""""}",https://api.taiga.io/api/v1/tasks?project=1,null,2024-01-21 08:00:00.000 +2,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":202,""ref"":2,""subject"":""Deploy to staging"",""status"":2,""status_extra_info"":{""name"":""Done""},""is_closed"":true,""created_date"":""2024-01-12T10:00:00.000Z"",""modified_date"":""2024-01-17T16:00:00.000Z"",""finished_date"":""2024-01-17T16:00:00.000Z"",""assigned_to"":7,""assigned_to_extra_info"":{""full_name_display"":""Carol White""},""user_story"":101,""milestone"":null,""is_blocked"":false,""blocked_note"":""""}",https://api.taiga.io/api/v1/tasks?project=1,null,2024-01-21 08:00:00.000 diff --git a/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_user_stories.csv b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_user_stories.csv new file mode 100644 index 00000000000..4bc83489eeb --- /dev/null +++ b/backend/plugins/taiga/e2e/raw_tables/_raw_taiga_api_user_stories.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":101,""ref"":1,""subject"":""Setup CI/CD Pipeline"",""status"":1,""status_extra_info"":{""name"":""In Progress""},""is_closed"":false,""created_date"":""2024-01-15T10:00:00.000Z"",""modified_date"":""2024-01-20T15:30:00.000Z"",""finish_date"":null,""assigned_to"":5,""assigned_to_extra_info"":{""full_name_display"":""Alice Smith""},""total_points"":8.0,""milestone"":1,""milestone_name"":""Sprint 1"",""priority"":3,""is_blocked"":false,""blocked_note"":""""}",https://api.taiga.io/api/v1/userstories?project=1,null,2024-01-21 08:00:00.000 +2,"{""ConnectionId"":1,""ProjectId"":1}","{""id"":102,""ref"":2,""subject"":""User Authentication"",""status"":2,""status_extra_info"":{""name"":""Done""},""is_closed"":true,""created_date"":""2024-01-10T09:00:00.000Z"",""modified_date"":""2024-01-18T17:00:00.000Z"",""finish_date"":""2024-01-18T17:00:00.000Z"",""assigned_to"":6,""assigned_to_extra_info"":{""full_name_display"":""Bob Jones""},""total_points"":5.0,""milestone"":null,""milestone_name"":null,""priority"":1,""is_blocked"":false,""blocked_note"":""""}",https://api.taiga.io/api/v1/userstories?project=1,null,2024-01-21 08:00:00.000 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_epics.csv b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_epics.csv new file mode 100644 index 00000000000..2cc0768c7ca --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_epics.csv @@ -0,0 +1,3 @@ +connection_id,project_id,epic_id,ref,subject,status,is_closed,created_date,modified_date,assigned_to,assigned_to_name,color +1,1,401,1,Platform v2.0,In Progress,0,2024-01-05T09:00:00.000+00:00,2024-01-20T10:00:00.000+00:00,5,Alice Smith,#F2711C +1,1,402,2,Mobile App,Open,0,2024-01-07T11:00:00.000+00:00,2024-01-18T09:00:00.000+00:00,0,,#2185D0 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_issues.csv b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_issues.csv new file mode 100644 index 00000000000..d4fa918fc72 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_issues.csv @@ -0,0 +1,3 @@ +connection_id,project_id,issue_id,ref,subject,status,issue_type_name,priority,severity,is_closed,created_date,modified_date,finished_date,assigned_to,assigned_to_name,milestone_id +1,1,301,1,Login page crash on mobile,New,Bug,High,Critical,0,2024-01-14T08:00:00.000+00:00,2024-01-19T14:00:00.000+00:00,,5,Alice Smith,0 +1,1,302,2,Add dark mode support,Closed,Enhancement,Normal,Minor,1,2024-01-08T10:00:00.000+00:00,2024-01-16T13:00:00.000+00:00,2024-01-16T13:00:00.000+00:00,6,Bob Jones,1 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_projects.csv b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_projects.csv new file mode 100644 index 00000000000..82e1575013d --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_projects.csv @@ -0,0 +1,2 @@ +connection_id,project_id,scope_config_id,name,slug,description,url,is_private,total_milestones,total_story_points +1,1,0,Test Project Alpha,test-project-alpha,A test project for the DevLake Taiga plugin e2e tests,,0,0,0 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_tasks.csv b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_tasks.csv new file mode 100644 index 00000000000..ad8131272fa --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_tasks.csv @@ -0,0 +1,3 @@ +connection_id,project_id,task_id,ref,subject,status,is_closed,created_date,modified_date,finished_date,assigned_to,assigned_to_name,user_story_id,milestone_id,is_blocked,blocked_note +1,1,201,1,Write unit tests,In Progress,0,2024-01-16T09:00:00.000+00:00,2024-01-21T11:00:00.000+00:00,,5,Alice Smith,101,1,0, +1,1,202,2,Deploy to staging,Done,1,2024-01-12T10:00:00.000+00:00,2024-01-17T16:00:00.000+00:00,2024-01-17T16:00:00.000+00:00,7,Carol White,101,0,0, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_user_stories.csv b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_user_stories.csv new file mode 100644 index 00000000000..ac053fec48a --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/_tool_taiga_user_stories.csv @@ -0,0 +1,3 @@ +connection_id,project_id,user_story_id,ref,subject,description,status,status_color,is_closed,created_date,modified_date,finished_date,assigned_to,assigned_to_name,total_points,milestone_id,milestone_name,priority,is_blocked,blocked_note +1,1,101,1,Setup CI/CD Pipeline,,In Progress,,0,2024-01-15T10:00:00.000+00:00,2024-01-20T15:30:00.000+00:00,,5,Alice Smith,8,1,Sprint 1,3,0, +1,1,102,2,User Authentication,,Done,,1,2024-01-10T09:00:00.000+00:00,2024-01-18T17:00:00.000+00:00,2024-01-18T17:00:00.000+00:00,6,Bob Jones,5,0,,1,0, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_epics.csv b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_epics.csv new file mode 100644 index 00000000000..118e5665334 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_epics.csv @@ -0,0 +1,3 @@ +board_id,issue_id +taiga:TaigaProject:1:1,taiga:TaigaEpic:1:401 +taiga:TaigaProject:1:1,taiga:TaigaEpic:1:402 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_issues.csv b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_issues.csv new file mode 100644 index 00000000000..afc67821289 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_issues.csv @@ -0,0 +1,3 @@ +board_id,issue_id +taiga:TaigaProject:1:1,taiga:TaigaIssue:1:301 +taiga:TaigaProject:1:1,taiga:TaigaIssue:1:302 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_tasks.csv b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_tasks.csv new file mode 100644 index 00000000000..0dddfd05a27 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_tasks.csv @@ -0,0 +1,3 @@ +board_id,issue_id +taiga:TaigaProject:1:1,taiga:TaigaTask:1:201 +taiga:TaigaProject:1:1,taiga:TaigaTask:1:202 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_user_stories.csv b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_user_stories.csv new file mode 100644 index 00000000000..d54c4ccb5e0 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/board_issues_from_user_stories.csv @@ -0,0 +1,3 @@ +board_id,issue_id +taiga:TaigaProject:1:1,taiga:TaigaUserStory:1:101 +taiga:TaigaProject:1:1,taiga:TaigaUserStory:1:102 diff --git a/backend/plugins/taiga/e2e/snapshot_tables/boards.csv b/backend/plugins/taiga/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..94046502c97 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +taiga:TaigaProject:1:1,Test Project Alpha,A test project for the DevLake Taiga plugin e2e tests,,, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/issues_from_epics.csv b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_epics.csv new file mode 100644 index 00000000000..bf58bfbae98 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_epics.csv @@ -0,0 +1,3 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +taiga:TaigaEpic:1:401,,,Platform v2.0,Platform v2.0,,,EPIC,Epic,In Progress,In Progress,,,2024-01-05T09:00:00.000+00:00,2024-01-20T10:00:00.000+00:00,,,,,,,,,,,,,,,0,, +taiga:TaigaEpic:1:402,,,Mobile App,Mobile App,,,EPIC,Epic,Open,Open,,,2024-01-07T11:00:00.000+00:00,2024-01-18T09:00:00.000+00:00,,,,,,,,,,,,,,,0,, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/issues_from_issues.csv b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_issues.csv new file mode 100644 index 00000000000..52da4dad986 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_issues.csv @@ -0,0 +1,3 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +taiga:TaigaIssue:1:301,,,Login page crash on mobile,Login page crash on mobile,,,BUG,Bug,New,New,,,2024-01-14T08:00:00.000+00:00,2024-01-19T14:00:00.000+00:00,,,,,,,,,,High,,,,,0,, +taiga:TaigaIssue:1:302,,,Add dark mode support,Add dark mode support,,,REQUIREMENT,Enhancement,Closed,Closed,,2024-01-16T13:00:00.000+00:00,2024-01-08T10:00:00.000+00:00,2024-01-16T13:00:00.000+00:00,,,,,,,,,,Normal,,,,,0,, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/issues_from_tasks.csv b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_tasks.csv new file mode 100644 index 00000000000..be3cfebb985 --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_tasks.csv @@ -0,0 +1,3 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +taiga:TaigaTask:1:201,,,Write unit tests,Write unit tests,,,TASK,Task,In Progress,In Progress,,,2024-01-16T09:00:00.000+00:00,2024-01-21T11:00:00.000+00:00,,,,,,,,,,,,,,,0,, +taiga:TaigaTask:1:202,,,Deploy to staging,Deploy to staging,,,TASK,Task,Done,Done,,2024-01-17T16:00:00.000+00:00,2024-01-12T10:00:00.000+00:00,2024-01-17T16:00:00.000+00:00,,,,,,,,,,,,,,,0,, diff --git a/backend/plugins/taiga/e2e/snapshot_tables/issues_from_user_stories.csv b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_user_stories.csv new file mode 100644 index 00000000000..bad31c7405c --- /dev/null +++ b/backend/plugins/taiga/e2e/snapshot_tables/issues_from_user_stories.csv @@ -0,0 +1,3 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +taiga:TaigaUserStory:1:101,,,#1,Setup CI/CD Pipeline,,,USER_STORY,User Story,TODO,In Progress,8,,2024-01-15T10:00:00.000+00:00,2024-01-20T15:30:00.000+00:00,,,,,,,5,Alice Smith,,,,,,,0,, +taiga:TaigaUserStory:1:102,,,#2,User Authentication,,,USER_STORY,User Story,DONE,Done,5,2024-01-18T17:00:00.000+00:00,2024-01-10T09:00:00.000+00:00,2024-01-18T17:00:00.000+00:00,12000,,,,,,6,Bob Jones,,,,,,,0,, diff --git a/backend/plugins/taiga/e2e/task_test.go b/backend/plugins/taiga/e2e/task_test.go new file mode 100644 index 00000000000..aefa1fb8fd2 --- /dev/null +++ b/backend/plugins/taiga/e2e/task_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +// TestTaigaTaskDataFlow verifies the full extract → convert pipeline for +// Taiga tasks: raw API JSON → _tool_taiga_tasks → ticket.Issue (type TASK). +// +// To regenerate snapshot CSVs from actual output, temporarily replace +// VerifyTableWithOptions with CreateSnapshot for each table. +func TestTaigaTaskDataFlow(t *testing.T) { + var taiga impl.Taiga + dataflowTester := e2ehelper.NewDataFlowTester(t, "taiga", taiga) + + taskData := &tasks.TaigaTaskData{ + Options: &tasks.TaigaOptions{ + ConnectionId: 1, + ProjectId: 1, + }, + } + + // ── Extraction: raw JSON → _tool_taiga_tasks ───────────────────────────── + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_taiga_api_tasks.csv", + "_raw_taiga_api_tasks", + ) + + dataflowTester.FlushTabler(&models.TaigaTask{}) + dataflowTester.Subtask(tasks.ExtractTasksMeta, taskData) + // Verify all tool-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(models.TaigaTask{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_taiga_tasks.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // ── Conversion: _tool_taiga_tasks → ticket.Issue / ticket.BoardIssue ───── + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertTasksMeta, taskData) + // Verify all domain-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_from_tasks.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues_from_tasks.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/taiga/e2e/user_story_test.go b/backend/plugins/taiga/e2e/user_story_test.go new file mode 100644 index 00000000000..1cb8553e028 --- /dev/null +++ b/backend/plugins/taiga/e2e/user_story_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +// TestTaigaUserStoryDataFlow verifies the full extract → convert pipeline +// for Taiga user stories: raw API JSON → _tool_taiga_user_stories → ticket.Issue. +// +// To regenerate snapshot CSVs from actual output, temporarily replace +// VerifyTableWithOptions with CreateSnapshot for each table. +func TestTaigaUserStoryDataFlow(t *testing.T) { + var taiga impl.Taiga + dataflowTester := e2ehelper.NewDataFlowTester(t, "taiga", taiga) + + taskData := &tasks.TaigaTaskData{ + Options: &tasks.TaigaOptions{ + ConnectionId: 1, + ProjectId: 1, + }, + } + + // ── Extraction: raw JSON → _tool_taiga_user_stories ───────────────────── + dataflowTester.ImportCsvIntoRawTable( + "./raw_tables/_raw_taiga_api_user_stories.csv", + "_raw_taiga_api_user_stories", + ) + + dataflowTester.FlushTabler(&models.TaigaUserStory{}) + dataflowTester.Subtask(tasks.ExtractUserStoriesMeta, taskData) + // Verify all tool-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(models.TaigaUserStory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_taiga_user_stories.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // ── Conversion: _tool_taiga_user_stories → ticket.Issue / ticket.BoardIssue ─ + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertUserStoriesMeta, taskData) + // Verify all domain-layer columns, ignoring NoPKModel timestamps and raw-data provenance. + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_from_user_stories.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues_from_user_stories.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/taiga/impl/impl.go b/backend/plugins/taiga/impl/impl.go new file mode 100644 index 00000000000..73e6e5b9fd7 --- /dev/null +++ b/backend/plugins/taiga/impl/impl.go @@ -0,0 +1,251 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" + "github.com/apache/incubator-devlake/plugins/taiga/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/taiga/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Taiga)(nil) + +type Taiga struct { +} + +func (p Taiga) Connection() dal.Tabler { + return &models.TaigaConnection{} +} + +func (p Taiga) Scope() plugin.ToolLayerScope { + return &models.TaigaProject{} +} + +func (p Taiga) ScopeConfig() dal.Tabler { + return &models.TaigaScopeConfig{} +} + +func (p Taiga) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Taiga) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.TaigaConnection{}, + &models.TaigaProject{}, + &models.TaigaUserStory{}, + &models.TaigaTask{}, + &models.TaigaIssue{}, + &models.TaigaEpic{}, + &models.TaigaScopeConfig{}, + } +} + +func (p Taiga) Description() string { + return "To collect and enrich data from Taiga" +} + +func (p Taiga) Name() string { + return "taiga" +} + +func (p Taiga) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectProjectsMeta, + tasks.ExtractProjectsMeta, + tasks.CollectUserStoriesMeta, + tasks.ExtractUserStoriesMeta, + tasks.CollectTasksMeta, + tasks.ExtractTasksMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, + tasks.CollectEpicsMeta, + tasks.ExtractEpicsMeta, + tasks.ConvertProjectsMeta, + tasks.ConvertUserStoriesMeta, + tasks.ConvertTasksMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertEpicsMeta, + } +} + +func (p Taiga) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.TaigaOptions + var err errors.Error + logger := taskCtx.GetLogger() + logger.Debug("%v", options) + + err = helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "could not decode Taiga options") + } + + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("taiga connectionId is invalid") + } + + connection := &models.TaigaConnection{} + connectionHelper := helper.NewConnectionHelper( + taskCtx, + nil, + p.Name(), + ) + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Taiga connection") + } + + taigaApiClient, err := tasks.NewTaigaApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create taiga api client") + } + + if op.ProjectId != 0 { + var scope *models.TaigaProject + db := taskCtx.GetDal() + err = db.First(&scope, dal.Where("connection_id = ? AND project_id = ?", op.ConnectionId, op.ProjectId)) + if err != nil && db.IsErrorNotFound(err) { + // Fetch from remote and save + // TODO: Implement fetching from Taiga API + return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project: %d", op.ProjectId)) + } + if err != nil { + return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find project: %d", op.ProjectId)) + } + if op.ScopeConfigId == 0 && scope.ScopeConfigId != 0 { + op.ScopeConfigId = scope.ScopeConfigId + } + } + + if op.ScopeConfig == nil && op.ScopeConfigId != 0 { + var scopeConfig models.TaigaScopeConfig + db := taskCtx.GetDal() + err = taskCtx.GetDal().First(&scopeConfig, dal.Where("id = ?", op.ScopeConfigId)) + if err != nil && db.IsErrorNotFound(err) { + return nil, errors.BadInput.Wrap(err, "fail to get scopeConfig") + } + op.ScopeConfig = &scopeConfig + if err != nil { + return nil, errors.BadInput.Wrap(err, "fail to make scopeConfig") + } + } + + if op.ScopeConfig == nil && op.ScopeConfigId == 0 { + op.ScopeConfig = new(models.TaigaScopeConfig) + } + + // Set default page size + if op.PageSize <= 0 || op.PageSize > 100 { + op.PageSize = 100 + } + + taskData := &tasks.TaigaTaskData{ + Options: &op, + ApiClient: taigaApiClient, + } + + return taskData, nil +} + +func (p Taiga) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Taiga) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/taiga" +} + +func (p Taiga) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Taiga) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.UpdateScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScope, + }, + "connections/:connectionId/scope-configs": { + "POST": api.CreateScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.UpdateScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Taiga) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.TaigaTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/taiga/models/connection.go b/backend/plugins/taiga/models/connection.go new file mode 100644 index 00000000000..e97979331fb --- /dev/null +++ b/backend/plugins/taiga/models/connection.go @@ -0,0 +1,143 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// TaigaConn holds the essential information to connect to the Taiga API +type TaigaConn struct { + helper.RestConnection `mapstructure:",squash"` + helper.BasicAuth `mapstructure:",squash"` + // Token is optional - can be provided directly or obtained via username/password auth + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (tc *TaigaConn) Sanitize() TaigaConn { + tc.Password = "" + tc.Token = utils.SanitizeString(tc.Token) + return *tc +} + +// SetupAuthentication sets up the HTTP request with authentication. +// If Token is set directly, use it. Otherwise exchange Username+Password for a token. +func (tc *TaigaConn) SetupAuthentication(req *http.Request) errors.Error { + if tc.Token != "" { + req.Header.Set("Authorization", "Bearer "+tc.Token) + return nil + } + if tc.Username != "" && tc.Password != "" { + token, err := tc.fetchToken() + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + } + return nil +} + +// fetchToken exchanges username+password for a Taiga auth token via POST /auth +func (tc *TaigaConn) fetchToken() (string, errors.Error) { + endpoint := strings.TrimSuffix(tc.Endpoint, "/") + // strip /api/v1 suffix to get base, then re-add /api/v1/auth + authURL := endpoint + "/auth" + + body, e := json.Marshal(map[string]string{ + "type": "normal", + "username": tc.Username, + "password": tc.Password, + }) + if e != nil { + return "", errors.Default.WrapRaw(e) + } + + resp, e := http.Post(authURL, "application/json", bytes.NewReader(body)) //nolint:noctx + if e != nil { + return "", errors.Default.WrapRaw(e) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.Default.New(fmt.Sprintf("taiga auth failed with status %d", resp.StatusCode)) + } + + var result map[string]interface{} + if e := json.NewDecoder(resp.Body).Decode(&result); e != nil { + return "", errors.Default.WrapRaw(e) + } + // Taiga returns auth_token (v5) or token (v6) + for _, key := range []string{"auth_token", "token"} { + if t, ok := result[key]; ok { + if token, ok := t.(string); ok && token != "" { + return token, nil + } + } + } + // fallback: read raw body hint + raw, _ := io.ReadAll(bytes.NewReader(body)) + return "", errors.Default.New(fmt.Sprintf("taiga auth response missing token field, body: %s", string(raw))) +} + +// TaigaConnection holds TaigaConn plus ID/Name for database storage +type TaigaConnection struct { + helper.BaseConnection `mapstructure:",squash"` + TaigaConn `mapstructure:",squash"` +} + +func (TaigaConnection) TableName() string { + return "_tool_taiga_connections" +} + +func (connection *TaigaConnection) MergeFromRequest(target *TaigaConnection, body map[string]interface{}) error { + token := target.Token + password := target.Password + + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + + modifiedToken := target.Token + modifiedPassword := target.Password + + // preserve existing token if not modified + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + + // preserve existing password if not modified + if modifiedPassword == "" || modifiedPassword == password { + target.Password = password + } + + return nil +} + +func (connection TaigaConnection) Sanitize() TaigaConnection { + connection.TaigaConn = connection.TaigaConn.Sanitize() + return connection +} diff --git a/backend/plugins/taiga/models/epic.go b/backend/plugins/taiga/models/epic.go new file mode 100644 index 00000000000..1c808b380f9 --- /dev/null +++ b/backend/plugins/taiga/models/epic.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// TaigaEpic represents an epic in Taiga +type TaigaEpic struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + EpicId uint64 `gorm:"primaryKey;autoIncrement:false" json:"id"` + Ref int `json:"ref"` + Subject string `gorm:"type:varchar(255)" json:"subject"` + Status string `gorm:"type:varchar(100)" json:"status"` + IsClosed bool `json:"isClosed"` + CreatedDate *time.Time `json:"createdDate"` + ModifiedDate *time.Time `json:"modifiedDate"` + AssignedTo uint64 `json:"assignedTo"` + AssignedToName string `gorm:"type:varchar(255)" json:"assignedToName"` + Color string `gorm:"type:varchar(20)" json:"color"` +} + +func (TaigaEpic) TableName() string { + return "_tool_taiga_epics" +} diff --git a/backend/plugins/taiga/models/issue.go b/backend/plugins/taiga/models/issue.go new file mode 100644 index 00000000000..90dda4f6503 --- /dev/null +++ b/backend/plugins/taiga/models/issue.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// TaigaIssue represents an issue (bug/enhancement/question) in Taiga +type TaigaIssue struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + IssueId uint64 `gorm:"primaryKey;autoIncrement:false" json:"id"` + Ref int `json:"ref"` + Subject string `gorm:"type:varchar(255)" json:"subject"` + Status string `gorm:"type:varchar(100)" json:"status"` + IssueTypeName string `gorm:"type:varchar(100)" json:"issueTypeName"` // e.g. "Bug", "Enhancement", "Question" + Priority string `gorm:"type:varchar(100)" json:"priority"` + Severity string `gorm:"type:varchar(100)" json:"severity"` + IsClosed bool `json:"isClosed"` + CreatedDate *time.Time `json:"createdDate"` + ModifiedDate *time.Time `json:"modifiedDate"` + FinishedDate *time.Time `json:"finishedDate"` + AssignedTo uint64 `json:"assignedTo"` + AssignedToName string `gorm:"type:varchar(255)" json:"assignedToName"` + MilestoneId uint64 `json:"milestoneId"` +} + +func (TaigaIssue) TableName() string { + return "_tool_taiga_issues" +} diff --git a/backend/plugins/taiga/models/migrationscripts/20250220_add_init_tables.go b/backend/plugins/taiga/models/migrationscripts/20250220_add_init_tables.go new file mode 100644 index 00000000000..3a0a6beeed3 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/20250220_add_init_tables.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type TaigaConnection20250220 struct { + ID uint64 `gorm:"primaryKey" json:"id"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + Endpoint string `json:"endpoint"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + Proxy string `json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` +} + +func (TaigaConnection20250220) TableName() string { + return "_tool_taiga_connections" +} + +type TaigaProject20250220 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"primaryKey;autoIncrement:false"` + Name string `gorm:"type:varchar(255)"` + Slug string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + Url string `gorm:"type:varchar(255)"` + IsPrivate bool + TotalMilestones int + TotalStoryPoints float64 + CreatedAt time.Time + UpdatedAt time.Time + RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"` + RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"` + RawDataId uint64 `gorm:"column:_raw_data_id"` + RawDataRemark string `gorm:"column:_raw_data_remark"` + ScopeConfigId uint64 +} + +func (TaigaProject20250220) TableName() string { + return "_tool_taiga_projects" +} + +type TaigaScopeConfig20250220 struct { + ID uint64 `gorm:"primaryKey;autoIncrement" json:"id"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + Entities string `gorm:"type:json" json:"entities"` +} + +func (TaigaScopeConfig20250220) TableName() string { + return "_tool_taiga_scope_configs" +} + +type TaigaUserStory20250220 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + UserStoryId uint64 `gorm:"primaryKey;autoIncrement:false"` + Ref int + Subject string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + Status string `gorm:"type:varchar(100)"` + StatusColor string `gorm:"type:varchar(20)"` + IsClosed bool + CreatedDate *time.Time + ModifiedDate *time.Time + FinishedDate *time.Time + AssignedTo uint64 + AssignedToName string `gorm:"type:varchar(255)"` + TotalPoints float64 + MilestoneId uint64 + MilestoneName string `gorm:"type:varchar(255)"` + Priority int + IsBlocked bool + BlockedNote string `gorm:"type:text"` + CreatedAt time.Time + UpdatedAt time.Time + RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"` + RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"` + RawDataId uint64 `gorm:"column:_raw_data_id"` + RawDataRemark string `gorm:"column:_raw_data_remark"` +} + +func (TaigaUserStory20250220) TableName() string { + return "_tool_taiga_user_stories" +} + +type addInitTables20250220 struct{} + +func (*addInitTables20250220) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &TaigaConnection20250220{}, + &TaigaProject20250220{}, + &TaigaScopeConfig20250220{}, + &TaigaUserStory20250220{}, + ) +} + +func (*addInitTables20250220) Version() uint64 { + return 20250220000001 +} + +func (*addInitTables20250220) Name() string { + return "Taiga init schemas" +} diff --git a/backend/plugins/taiga/models/migrationscripts/20260306_add_task_issue_epic_tables.go b/backend/plugins/taiga/models/migrationscripts/20260306_add_task_issue_epic_tables.go new file mode 100644 index 00000000000..ff84ba19b84 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/20260306_add_task_issue_epic_tables.go @@ -0,0 +1,128 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type TaigaTask20260306 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + TaskId uint64 `gorm:"primaryKey;autoIncrement:false"` + Ref int + Subject string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + IsClosed bool + CreatedDate *time.Time + ModifiedDate *time.Time + FinishedDate *time.Time + AssignedTo uint64 + AssignedToName string `gorm:"type:varchar(255)"` + UserStoryId uint64 + MilestoneId uint64 + IsBlocked bool + BlockedNote string `gorm:"type:text"` + CreatedAt time.Time + UpdatedAt time.Time + RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"` + RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"` + RawDataId uint64 `gorm:"column:_raw_data_id"` + RawDataRemark string `gorm:"column:_raw_data_remark"` +} + +func (TaigaTask20260306) TableName() string { + return "_tool_taiga_tasks" +} + +type TaigaIssue20260306 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + IssueId uint64 `gorm:"primaryKey;autoIncrement:false"` + Ref int + Subject string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + IssueTypeName string `gorm:"type:varchar(100)"` + Priority string `gorm:"type:varchar(100)"` + Severity string `gorm:"type:varchar(100)"` + IsClosed bool + CreatedDate *time.Time + ModifiedDate *time.Time + FinishedDate *time.Time + AssignedTo uint64 + AssignedToName string `gorm:"type:varchar(255)"` + MilestoneId uint64 + CreatedAt time.Time + UpdatedAt time.Time + RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"` + RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"` + RawDataId uint64 `gorm:"column:_raw_data_id"` + RawDataRemark string `gorm:"column:_raw_data_remark"` +} + +func (TaigaIssue20260306) TableName() string { + return "_tool_taiga_issues" +} + +type TaigaEpic20260306 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + EpicId uint64 `gorm:"primaryKey;autoIncrement:false"` + Ref int + Subject string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + IsClosed bool + CreatedDate *time.Time + ModifiedDate *time.Time + AssignedTo uint64 + AssignedToName string `gorm:"type:varchar(255)"` + Color string `gorm:"type:varchar(20)"` + CreatedAt time.Time + UpdatedAt time.Time + RawDataParams string `gorm:"column:_raw_data_params;type:varchar(255);index"` + RawDataTable string `gorm:"column:_raw_data_table;type:varchar(255)"` + RawDataId uint64 `gorm:"column:_raw_data_id"` + RawDataRemark string `gorm:"column:_raw_data_remark"` +} + +func (TaigaEpic20260306) TableName() string { + return "_tool_taiga_epics" +} + +type addTaskIssueEpicTables20260306 struct{} + +func (*addTaskIssueEpicTables20260306) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &TaigaTask20260306{}, + &TaigaIssue20260306{}, + &TaigaEpic20260306{}, + ) +} + +func (*addTaskIssueEpicTables20260306) Version() uint64 { + return 20260306000001 +} + +func (*addTaskIssueEpicTables20260306) Name() string { + return "Taiga add task, issue, epic tables" +} diff --git a/backend/plugins/taiga/models/migrationscripts/register.go b/backend/plugins/taiga/models/migrationscripts/register.go new file mode 100644 index 00000000000..d2cfd08e269 --- /dev/null +++ b/backend/plugins/taiga/models/migrationscripts/register.go @@ -0,0 +1,30 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables20250220), + new(addTaskIssueEpicTables20260306), + } +} diff --git a/backend/plugins/taiga/models/project.go b/backend/plugins/taiga/models/project.go new file mode 100644 index 00000000000..7b26125624e --- /dev/null +++ b/backend/plugins/taiga/models/project.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +// TaigaProject represents a Taiga project (scope) +type TaigaProject struct { + common.Scope `mapstructure:",squash"` + ProjectId uint64 `json:"projectId" gorm:"primaryKey;autoIncrement:false"` + Name string `gorm:"type:varchar(255)" json:"name"` + Slug string `gorm:"type:varchar(255)" json:"slug"` + Description string `gorm:"type:text" json:"description"` + Url string `gorm:"type:varchar(255)" json:"url"` + IsPrivate bool `json:"isPrivate"` + TotalMilestones int `json:"totalMilestones"` + TotalStoryPoints float64 `json:"totalStoryPoints"` +} + +func (p TaigaProject) ScopeId() string { + return fmt.Sprintf("%d", p.ProjectId) +} + +func (p TaigaProject) ScopeName() string { + return p.Name +} + +func (p TaigaProject) ScopeFullName() string { + return p.Name +} + +func (p TaigaProject) ScopeParams() interface{} { + return &plugin.ApiResourceInput{ + Params: map[string]string{ + "connectionId": fmt.Sprintf("%d", p.ConnectionId), + "projectId": fmt.Sprintf("%d", p.ProjectId), + }, + } +} + +func (TaigaProject) TableName() string { + return "_tool_taiga_projects" +} diff --git a/backend/plugins/taiga/models/scope_config.go b/backend/plugins/taiga/models/scope_config.go new file mode 100644 index 00000000000..c6397083bca --- /dev/null +++ b/backend/plugins/taiga/models/scope_config.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" +) + +type StatusMapping struct { + StandardStatus string `json:"standardStatus"` +} + +type StatusMappings map[string]StatusMapping + +type TypeMapping struct { + StandardType string `json:"standardType"` + StatusMappings StatusMappings `json:"statusMappings"` +} + +type TaigaScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + TypeMappings map[string]TypeMapping `mapstructure:"typeMappings,omitempty" json:"typeMappings" gorm:"type:json;serializer:json"` +} + +func (r *TaigaScopeConfig) SetConnectionId(c *TaigaScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} + +func (r *TaigaScopeConfig) Validate() errors.Error { + // Add validation logic if needed + return nil +} + +func (r TaigaScopeConfig) TableName() string { + return "_tool_taiga_scope_configs" +} diff --git a/backend/plugins/taiga/models/task.go b/backend/plugins/taiga/models/task.go new file mode 100644 index 00000000000..248267cf9c2 --- /dev/null +++ b/backend/plugins/taiga/models/task.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// TaigaTask represents a task (sub-task of a user story) in Taiga +type TaigaTask struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + TaskId uint64 `gorm:"primaryKey;autoIncrement:false" json:"id"` + Ref int `json:"ref"` + Subject string `gorm:"type:varchar(255)" json:"subject"` + Status string `gorm:"type:varchar(100)" json:"status"` + IsClosed bool `json:"isClosed"` + CreatedDate *time.Time `json:"createdDate"` + ModifiedDate *time.Time `json:"modifiedDate"` + FinishedDate *time.Time `json:"finishedDate"` + AssignedTo uint64 `json:"assignedTo"` + AssignedToName string `gorm:"type:varchar(255)" json:"assignedToName"` + UserStoryId uint64 `json:"userStoryId"` + MilestoneId uint64 `json:"milestoneId"` + IsBlocked bool `json:"isBlocked"` + BlockedNote string `gorm:"type:text" json:"blockedNote"` +} + +func (TaigaTask) TableName() string { + return "_tool_taiga_tasks" +} diff --git a/backend/plugins/taiga/models/user_story.go b/backend/plugins/taiga/models/user_story.go new file mode 100644 index 00000000000..498584477dc --- /dev/null +++ b/backend/plugins/taiga/models/user_story.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// TaigaUserStory represents a user story in Taiga +type TaigaUserStory struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ProjectId uint64 `gorm:"index"` + UserStoryId uint64 `gorm:"primaryKey;autoIncrement:false" json:"id"` + Ref int `json:"ref"` + Subject string `gorm:"type:varchar(255)" json:"subject"` + Description string `gorm:"type:text" json:"description"` + Status string `gorm:"type:varchar(100)" json:"status"` + StatusColor string `gorm:"type:varchar(20)" json:"statusColor"` + IsClosed bool `json:"isClosed"` + CreatedDate *time.Time `json:"createdDate"` + ModifiedDate *time.Time `json:"modifiedDate"` + FinishedDate *time.Time `json:"finishedDate"` + AssignedTo uint64 `json:"assignedTo"` + AssignedToName string `gorm:"type:varchar(255)" json:"assignedToName"` + TotalPoints float64 `json:"totalPoints"` + MilestoneId uint64 `json:"milestoneId"` + MilestoneName string `gorm:"type:varchar(255)" json:"milestoneName"` + Priority int `json:"priority"` + IsBlocked bool `json:"isBlocked"` + BlockedNote string `gorm:"type:text" json:"blockedNote"` +} + +func (TaigaUserStory) TableName() string { + return "_tool_taiga_user_stories" +} diff --git a/backend/plugins/taiga/taiga.go b/backend/plugins/taiga/taiga.go new file mode 100644 index 00000000000..4a6da00a8a1 --- /dev/null +++ b/backend/plugins/taiga/taiga.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/taiga/impl" + "github.com/spf13/cobra" +) + +// PluginEntry exports a symbol for Framework to load +var PluginEntry impl.Taiga //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "taiga"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "taiga connection id") + projectId := cmd.Flags().Uint64P("projectId", "p", 0, "taiga project id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("projectId") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "projectId": *projectId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/taiga/tasks/api_client.go b/backend/plugins/taiga/tasks/api_client.go new file mode 100644 index 00000000000..4b722cc1a6c --- /dev/null +++ b/backend/plugins/taiga/tasks/api_client.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +func NewTaigaApiClient(taskCtx plugin.TaskContext, connection *models.TaigaConnection) (*api.ApiAsyncClient, errors.Error) { + // create synchronize api client + apiClient, err := api.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + + // create rate limit calculator + rateLimiter := &api.ApiRateLimitCalculator{ + UserRateLimitPerHour: connection.RateLimitPerHour, + } + + asyncApiClient, err := api.CreateAsyncApiClient( + taskCtx, + apiClient, + rateLimiter, + ) + if err != nil { + return nil, err + } + + return asyncApiClient, nil +} diff --git a/backend/plugins/taiga/tasks/epic_collector.go b/backend/plugins/taiga/tasks/epic_collector.go new file mode 100644 index 00000000000..f99d48aabee --- /dev/null +++ b/backend/plugins/taiga/tasks/epic_collector.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_EPIC_TABLE = "taiga_api_epics" + +var _ plugin.SubTaskEntryPoint = CollectEpics + +var CollectEpicsMeta = plugin.SubTaskMeta{ + Name: "collectEpics", + EntryPoint: CollectEpics, + EnabledByDefault: true, + Description: "collect Taiga epics", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectEpics(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect epics") + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_EPIC_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 1000, + UrlTemplate: "epics", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var result []json.RawMessage + err := api.UnmarshalResponse(res, &result) + if err != nil { + return nil, err + } + return result, nil + }, + }) + if err != nil { + logger.Error(err, "collect epics error") + return err + } + return collector.Execute() +} diff --git a/backend/plugins/taiga/tasks/epic_convertor.go b/backend/plugins/taiga/tasks/epic_convertor.go new file mode 100644 index 00000000000..ed5bb53b0fb --- /dev/null +++ b/backend/plugins/taiga/tasks/epic_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var ConvertEpicsMeta = plugin.SubTaskMeta{ + Name: "convertEpics", + EntryPoint: ConvertEpics, + EnabledByDefault: true, + Description: "convert Taiga epics", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertEpics(subtaskCtx plugin.SubTaskContext) errors.Error { + logger := subtaskCtx.GetLogger() + data := subtaskCtx.GetData().(*TaigaTaskData) + db := subtaskCtx.GetDal() + + epicIdGen := didgen.NewDomainIdGenerator(&models.TaigaEpic{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.TaigaEpic]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: subtaskCtx, + Table: RAW_EPIC_TABLE, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.TaigaEpic{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(epic *models.TaigaEpic) ([]interface{}, errors.Error) { + var result []interface{} + + issue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: epicIdGen.Generate(epic.ConnectionId, epic.EpicId), + }, + IssueKey: epic.Subject, + Title: epic.Subject, + Type: "EPIC", + OriginalType: "Epic", + Status: epic.Status, + OriginalStatus: epic.Status, + CreatedDate: epic.CreatedDate, + UpdatedDate: epic.ModifiedDate, + } + + result = append(result, issue) + + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: issue.Id, + } + result = append(result, boardIssue) + + logger.Debug("converted epic %d", epic.EpicId) + return result, nil + }, + }) + + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/taiga/tasks/epic_extractor.go b/backend/plugins/taiga/tasks/epic_extractor.go new file mode 100644 index 00000000000..6acd76dacf9 --- /dev/null +++ b/backend/plugins/taiga/tasks/epic_extractor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractEpics + +var ExtractEpicsMeta = plugin.SubTaskMeta{ + Name: "extractEpics", + EntryPoint: ExtractEpics, + EnabledByDefault: true, + Description: "extract Taiga epics", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractEpics(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_EPIC_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiEpic struct { + Id uint64 `json:"id"` + Ref int `json:"ref"` + Subject string `json:"subject"` + StatusExtraInfo struct { + Name string `json:"name"` + } `json:"status_extra_info"` + IsClosed bool `json:"is_closed"` + CreatedDate *time.Time `json:"created_date"` + ModifiedDate *time.Time `json:"modified_date"` + AssignedTo *uint64 `json:"assigned_to"` + AssignedToExtraInfo *struct { + FullNameDisplay string `json:"full_name_display"` + } `json:"assigned_to_extra_info"` + Color string `json:"color"` + } + err := json.Unmarshal(row.Data, &apiEpic) + if err != nil { + return nil, errors.Default.Wrap(err, "error unmarshalling epic") + } + + var assignedTo uint64 + var assignedToName string + if apiEpic.AssignedTo != nil { + assignedTo = *apiEpic.AssignedTo + } + if apiEpic.AssignedToExtraInfo != nil { + assignedToName = apiEpic.AssignedToExtraInfo.FullNameDisplay + } + + epic := &models.TaigaEpic{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + EpicId: apiEpic.Id, + Ref: apiEpic.Ref, + Subject: apiEpic.Subject, + Status: apiEpic.StatusExtraInfo.Name, + IsClosed: apiEpic.IsClosed, + CreatedDate: apiEpic.CreatedDate, + ModifiedDate: apiEpic.ModifiedDate, + AssignedTo: assignedTo, + AssignedToName: assignedToName, + Color: apiEpic.Color, + } + + return []interface{}{epic}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/taiga/tasks/issue_collector.go b/backend/plugins/taiga/tasks/issue_collector.go new file mode 100644 index 00000000000..cf79d539901 --- /dev/null +++ b/backend/plugins/taiga/tasks/issue_collector.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_ISSUE_TABLE = "taiga_api_issues" + +var _ plugin.SubTaskEntryPoint = CollectIssues + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "collectIssues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "collect Taiga issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect issues") + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_ISSUE_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 1000, + UrlTemplate: "issues", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var result []json.RawMessage + err := api.UnmarshalResponse(res, &result) + if err != nil { + return nil, err + } + return result, nil + }, + }) + if err != nil { + logger.Error(err, "collect issues error") + return err + } + return collector.Execute() +} diff --git a/backend/plugins/taiga/tasks/issue_convertor.go b/backend/plugins/taiga/tasks/issue_convertor.go new file mode 100644 index 00000000000..5612a3b588c --- /dev/null +++ b/backend/plugins/taiga/tasks/issue_convertor.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "strings" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "convertIssues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "convert Taiga issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// taigaIssueTypeToDevLake maps Taiga's customizable issue type names to +// DevLake standard issue types. Taiga's default types are Bug, Enhancement, +// and Question; additional custom types fall back to REQUIREMENT. +func taigaIssueTypeToDevLake(typeName string) string { + switch strings.ToLower(typeName) { + case "bug": + return "BUG" + case "enhancement", "feature": + return "REQUIREMENT" + case "question": + return "QUESTION" + default: + return "REQUIREMENT" + } +} + +func ConvertIssues(subtaskCtx plugin.SubTaskContext) errors.Error { + logger := subtaskCtx.GetLogger() + data := subtaskCtx.GetData().(*TaigaTaskData) + db := subtaskCtx.GetDal() + + issueIdGen := didgen.NewDomainIdGenerator(&models.TaigaIssue{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.TaigaIssue]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: subtaskCtx, + Table: RAW_ISSUE_TABLE, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.TaigaIssue{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(taigaIssue *models.TaigaIssue) ([]interface{}, errors.Error) { + var result []interface{} + + devLakeType := taigaIssueTypeToDevLake(taigaIssue.IssueTypeName) + originalType := taigaIssue.IssueTypeName + if originalType == "" { + originalType = "Issue" + } + + issue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: issueIdGen.Generate(taigaIssue.ConnectionId, taigaIssue.IssueId), + }, + IssueKey: taigaIssue.Subject, + Title: taigaIssue.Subject, + Type: devLakeType, + OriginalType: originalType, + Status: taigaIssue.Status, + OriginalStatus: taigaIssue.Status, + Priority: taigaIssue.Priority, + CreatedDate: taigaIssue.CreatedDate, + UpdatedDate: taigaIssue.ModifiedDate, + ResolutionDate: taigaIssue.FinishedDate, + } + + result = append(result, issue) + + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: issue.Id, + } + result = append(result, boardIssue) + + logger.Debug("converted issue %d (type: %s → %s)", taigaIssue.IssueId, originalType, devLakeType) + return result, nil + }, + }) + + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/taiga/tasks/issue_extractor.go b/backend/plugins/taiga/tasks/issue_extractor.go new file mode 100644 index 00000000000..f2dd54ca995 --- /dev/null +++ b/backend/plugins/taiga/tasks/issue_extractor.go @@ -0,0 +1,136 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "extractIssues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "extract Taiga issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_ISSUE_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiIssue struct { + Id uint64 `json:"id"` + Ref int `json:"ref"` + Subject string `json:"subject"` + StatusExtraInfo struct { + Name string `json:"name"` + } `json:"status_extra_info"` + TypeExtraInfo *struct { + Name string `json:"name"` + } `json:"type_extra_info"` + PriorityExtraInfo *struct { + Name string `json:"name"` + } `json:"priority_extra_info"` + SeverityExtraInfo *struct { + Name string `json:"name"` + } `json:"severity_extra_info"` + IsClosed bool `json:"is_closed"` + CreatedDate *time.Time `json:"created_date"` + ModifiedDate *time.Time `json:"modified_date"` + FinishedDate *time.Time `json:"finished_date"` + AssignedTo *uint64 `json:"assigned_to"` + AssignedToExtraInfo *struct { + FullNameDisplay string `json:"full_name_display"` + } `json:"assigned_to_extra_info"` + Milestone *uint64 `json:"milestone"` + } + err := json.Unmarshal(row.Data, &apiIssue) + if err != nil { + return nil, errors.Default.Wrap(err, "error unmarshalling issue") + } + + var assignedTo uint64 + var assignedToName string + if apiIssue.AssignedTo != nil { + assignedTo = *apiIssue.AssignedTo + } + if apiIssue.AssignedToExtraInfo != nil { + assignedToName = apiIssue.AssignedToExtraInfo.FullNameDisplay + } + var issueTypeName string + if apiIssue.TypeExtraInfo != nil { + issueTypeName = apiIssue.TypeExtraInfo.Name + } + var priority string + if apiIssue.PriorityExtraInfo != nil { + priority = apiIssue.PriorityExtraInfo.Name + } + var severity string + if apiIssue.SeverityExtraInfo != nil { + severity = apiIssue.SeverityExtraInfo.Name + } + var milestoneId uint64 + if apiIssue.Milestone != nil { + milestoneId = *apiIssue.Milestone + } + + issue := &models.TaigaIssue{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + IssueId: apiIssue.Id, + Ref: apiIssue.Ref, + Subject: apiIssue.Subject, + Status: apiIssue.StatusExtraInfo.Name, + IssueTypeName: issueTypeName, + Priority: priority, + Severity: severity, + IsClosed: apiIssue.IsClosed, + CreatedDate: apiIssue.CreatedDate, + ModifiedDate: apiIssue.ModifiedDate, + FinishedDate: apiIssue.FinishedDate, + AssignedTo: assignedTo, + AssignedToName: assignedToName, + MilestoneId: milestoneId, + } + + return []interface{}{issue}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/taiga/tasks/project_collector.go b/backend/plugins/taiga/tasks/project_collector.go new file mode 100644 index 00000000000..4a6462b40f6 --- /dev/null +++ b/backend/plugins/taiga/tasks/project_collector.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_PROJECT_TABLE = "taiga_api_projects" + +var _ plugin.SubTaskEntryPoint = CollectProjects + +var CollectProjectsMeta = plugin.SubTaskMeta{ + Name: "collectProjects", + EntryPoint: CollectProjects, + EnabledByDefault: true, + Description: "collect Taiga projects", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectProjects(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect projects") + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_PROJECT_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "projects/{{ .Params.ProjectId }}", + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var result json.RawMessage + err := api.UnmarshalResponse(res, &result) + if err != nil { + return nil, err + } + return []json.RawMessage{result}, nil + }, + }) + if err != nil { + logger.Error(err, "collect project error") + return err + } + return collector.Execute() +} + +type TaigaApiParams struct { + ConnectionId uint64 + ProjectId uint64 +} diff --git a/backend/plugins/taiga/tasks/project_convertor.go b/backend/plugins/taiga/tasks/project_convertor.go new file mode 100644 index 00000000000..032831d0717 --- /dev/null +++ b/backend/plugins/taiga/tasks/project_convertor.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var ConvertProjectsMeta = plugin.SubTaskMeta{ + Name: "convertProjects", + EntryPoint: ConvertProjects, + EnabledByDefault: true, + Description: "convert Taiga projects", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertProjects(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + logger.Info("convert project:%d", data.Options.ProjectId) + + idGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.TaigaProject{}), + dal.Where("connection_id = ? AND project_id = ?", data.Options.ConnectionId, data.Options.ProjectId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_PROJECT_TABLE, + }, + InputRowType: reflect.TypeOf(models.TaigaProject{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + project := inputRow.(*models.TaigaProject) + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: idGen.Generate(data.Options.ConnectionId, data.Options.ProjectId)}, + Name: project.Name, + Description: project.Description, + Url: project.Url, + } + return []interface{}{ + domainBoard, + }, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/taiga/tasks/project_extractor.go b/backend/plugins/taiga/tasks/project_extractor.go new file mode 100644 index 00000000000..669e34760a8 --- /dev/null +++ b/backend/plugins/taiga/tasks/project_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractProjects + +var ExtractProjectsMeta = plugin.SubTaskMeta{ + Name: "extractProjects", + EntryPoint: ExtractProjects, + EnabledByDefault: true, + Description: "extract Taiga projects", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractProjects(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_PROJECT_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiProject struct { + Id uint64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` + Description string `json:"description"` + CreatedDate string `json:"created_date"` + ModifiedDate string `json:"modified_date"` + } + err := json.Unmarshal(row.Data, &apiProject) + if err != nil { + return nil, errors.Default.Wrap(err, "error unmarshalling project") + } + + project := &models.TaigaProject{ + Scope: common.Scope{ + ConnectionId: data.Options.ConnectionId, + }, + ProjectId: apiProject.Id, + Name: apiProject.Name, + Slug: apiProject.Slug, + Description: apiProject.Description, + } + + return []interface{}{project}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/taiga/tasks/task_collector.go b/backend/plugins/taiga/tasks/task_collector.go new file mode 100644 index 00000000000..fabec6e39a4 --- /dev/null +++ b/backend/plugins/taiga/tasks/task_collector.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_TASK_TABLE = "taiga_api_tasks" + +var _ plugin.SubTaskEntryPoint = CollectTasks + +var CollectTasksMeta = plugin.SubTaskMeta{ + Name: "collectTasks", + EntryPoint: CollectTasks, + EnabledByDefault: true, + Description: "collect Taiga tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect tasks") + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_TASK_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 1000, + UrlTemplate: "tasks", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var result []json.RawMessage + err := api.UnmarshalResponse(res, &result) + if err != nil { + return nil, err + } + return result, nil + }, + }) + if err != nil { + logger.Error(err, "collect tasks error") + return err + } + return collector.Execute() +} diff --git a/backend/plugins/taiga/tasks/task_convertor.go b/backend/plugins/taiga/tasks/task_convertor.go new file mode 100644 index 00000000000..f13b51686db --- /dev/null +++ b/backend/plugins/taiga/tasks/task_convertor.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var ConvertTasksMeta = plugin.SubTaskMeta{ + Name: "convertTasks", + EntryPoint: ConvertTasks, + EnabledByDefault: true, + Description: "convert Taiga tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertTasks(subtaskCtx plugin.SubTaskContext) errors.Error { + logger := subtaskCtx.GetLogger() + data := subtaskCtx.GetData().(*TaigaTaskData) + db := subtaskCtx.GetDal() + + taskIdGen := didgen.NewDomainIdGenerator(&models.TaigaTask{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.TaigaTask]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: subtaskCtx, + Table: RAW_TASK_TABLE, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.TaigaTask{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(task *models.TaigaTask) ([]interface{}, errors.Error) { + var result []interface{} + + issue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: taskIdGen.Generate(task.ConnectionId, task.TaskId), + }, + IssueKey: task.Subject, + Title: task.Subject, + Type: "TASK", + OriginalType: "Task", + Status: task.Status, + OriginalStatus: task.Status, + CreatedDate: task.CreatedDate, + UpdatedDate: task.ModifiedDate, + ResolutionDate: task.FinishedDate, + } + + result = append(result, issue) + + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: issue.Id, + } + result = append(result, boardIssue) + + logger.Debug("converted task %d", task.TaskId) + return result, nil + }, + }) + + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/taiga/tasks/task_data.go b/backend/plugins/taiga/tasks/task_data.go new file mode 100644 index 00000000000..dd6b3d58bb0 --- /dev/null +++ b/backend/plugins/taiga/tasks/task_data.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +type TaigaOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` + ProjectId uint64 `json:"projectId" mapstructure:"projectId"` + ScopeConfig *models.TaigaScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId"` + PageSize int `json:"pageSize" mapstructure:"pageSize"` +} + +type TaigaTaskData struct { + Options *TaigaOptions + ApiClient *api.ApiAsyncClient +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*TaigaOptions, errors.Error) { + var op TaigaOptions + err := api.Decode(options, &op, nil) + if err != nil { + return nil, err + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New(fmt.Sprintf("invalid connectionId:%d", op.ConnectionId)) + } + if op.ProjectId == 0 { + return nil, errors.BadInput.New(fmt.Sprintf("invalid projectId:%d", op.ProjectId)) + } + return &op, nil +} diff --git a/backend/plugins/taiga/tasks/task_extractor.go b/backend/plugins/taiga/tasks/task_extractor.go new file mode 100644 index 00000000000..29bece08e12 --- /dev/null +++ b/backend/plugins/taiga/tasks/task_extractor.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTasks + +var ExtractTasksMeta = plugin.SubTaskMeta{ + Name: "extractTasks", + EntryPoint: ExtractTasks, + EnabledByDefault: true, + Description: "extract Taiga tasks", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractTasks(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_TASK_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiTask struct { + Id uint64 `json:"id"` + Ref int `json:"ref"` + Subject string `json:"subject"` + StatusExtraInfo struct { + Name string `json:"name"` + } `json:"status_extra_info"` + IsClosed bool `json:"is_closed"` + CreatedDate *time.Time `json:"created_date"` + ModifiedDate *time.Time `json:"modified_date"` + FinishedDate *time.Time `json:"finished_date"` + AssignedTo *uint64 `json:"assigned_to"` + AssignedToExtraInfo *struct { + FullNameDisplay string `json:"full_name_display"` + } `json:"assigned_to_extra_info"` + UserStory *uint64 `json:"user_story"` + Milestone *uint64 `json:"milestone"` + IsBlocked bool `json:"is_blocked"` + BlockedNote string `json:"blocked_note"` + } + err := json.Unmarshal(row.Data, &apiTask) + if err != nil { + return nil, errors.Default.Wrap(err, "error unmarshalling task") + } + + var assignedTo uint64 + var assignedToName string + if apiTask.AssignedTo != nil { + assignedTo = *apiTask.AssignedTo + } + if apiTask.AssignedToExtraInfo != nil { + assignedToName = apiTask.AssignedToExtraInfo.FullNameDisplay + } + var userStoryId uint64 + if apiTask.UserStory != nil { + userStoryId = *apiTask.UserStory + } + var milestoneId uint64 + if apiTask.Milestone != nil { + milestoneId = *apiTask.Milestone + } + + task := &models.TaigaTask{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + TaskId: apiTask.Id, + Ref: apiTask.Ref, + Subject: apiTask.Subject, + Status: apiTask.StatusExtraInfo.Name, + IsClosed: apiTask.IsClosed, + CreatedDate: apiTask.CreatedDate, + ModifiedDate: apiTask.ModifiedDate, + FinishedDate: apiTask.FinishedDate, + AssignedTo: assignedTo, + AssignedToName: assignedToName, + UserStoryId: userStoryId, + MilestoneId: milestoneId, + IsBlocked: apiTask.IsBlocked, + BlockedNote: apiTask.BlockedNote, + } + + return []interface{}{task}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/taiga/tasks/user_story_collector.go b/backend/plugins/taiga/tasks/user_story_collector.go new file mode 100644 index 00000000000..80f3ec7a67d --- /dev/null +++ b/backend/plugins/taiga/tasks/user_story_collector.go @@ -0,0 +1,80 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const RAW_USER_STORY_TABLE = "taiga_api_user_stories" + +var _ plugin.SubTaskEntryPoint = CollectUserStories + +var CollectUserStoriesMeta = plugin.SubTaskMeta{ + Name: "collectUserStories", + EntryPoint: CollectUserStories, + EnabledByDefault: true, + Description: "collect Taiga user stories", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectUserStories(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect user stories") + + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_USER_STORY_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 1000, // Fetch all in one page - Taiga returns all user stories for a project + UrlTemplate: "userstories", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + query.Set("project", fmt.Sprintf("%d", data.Options.ProjectId)) + // Don't specify page - get all results + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var result []json.RawMessage + err := api.UnmarshalResponse(res, &result) + if err != nil { + return nil, err + } + return result, nil + }, + }) + if err != nil { + logger.Error(err, "collect user stories error") + return err + } + return collector.Execute() +} diff --git a/backend/plugins/taiga/tasks/user_story_convertor.go b/backend/plugins/taiga/tasks/user_story_convertor.go new file mode 100644 index 00000000000..963a25ae18e --- /dev/null +++ b/backend/plugins/taiga/tasks/user_story_convertor.go @@ -0,0 +1,129 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var ConvertUserStoriesMeta = plugin.SubTaskMeta{ + Name: "convertUserStories", + EntryPoint: ConvertUserStories, + EnabledByDefault: true, + Description: "convert Taiga user stories", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertUserStories(subtaskCtx plugin.SubTaskContext) errors.Error { + logger := subtaskCtx.GetLogger() + data := subtaskCtx.GetData().(*TaigaTaskData) + db := subtaskCtx.GetDal() + + issueIdGen := didgen.NewDomainIdGenerator(&models.TaigaUserStory{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.TaigaProject{}) + boardId := boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.TaigaUserStory]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: subtaskCtx, + Table: RAW_USER_STORY_TABLE, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.TaigaUserStory{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(userStory *models.TaigaUserStory) ([]interface{}, errors.Error) { + var result []interface{} + + // Map Taiga is_closed to DevLake standard status + status := "TODO" + if userStory.IsClosed { + status = "DONE" + } + + issue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{ + Id: issueIdGen.Generate(userStory.ConnectionId, userStory.UserStoryId), + }, + IssueKey: fmt.Sprintf("#%d", userStory.Ref), + Title: userStory.Subject, + Type: "USER_STORY", + OriginalType: "User Story", + Status: status, + OriginalStatus: userStory.Status, + CreatedDate: userStory.CreatedDate, + UpdatedDate: userStory.ModifiedDate, + ResolutionDate: userStory.FinishedDate, + AssigneeId: fmt.Sprintf("%d", userStory.AssignedTo), + AssigneeName: userStory.AssignedToName, + } + + if userStory.TotalPoints > 0 { + issue.StoryPoint = &userStory.TotalPoints + } + + // Calculate lead time: creation → resolution for closed stories + if userStory.IsClosed && issue.CreatedDate != nil && issue.ResolutionDate != nil { + leadTimeMinutes := uint(issue.ResolutionDate.Sub(*issue.CreatedDate).Minutes()) + if leadTimeMinutes > 0 { + issue.LeadTimeMinutes = &leadTimeMinutes + } + } + + result = append(result, issue) + + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: issue.Id, + } + result = append(result, boardIssue) + + logger.Debug("converted user story %d", userStory.UserStoryId) + return result, nil + }, + }) + + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/taiga/tasks/user_story_extractor.go b/backend/plugins/taiga/tasks/user_story_extractor.go new file mode 100644 index 00000000000..642981e2076 --- /dev/null +++ b/backend/plugins/taiga/tasks/user_story_extractor.go @@ -0,0 +1,135 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/taiga/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractUserStories + +var ExtractUserStoriesMeta = plugin.SubTaskMeta{ + Name: "extractUserStories", + EntryPoint: ExtractUserStories, + EnabledByDefault: true, + Description: "extract Taiga user stories", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractUserStories(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TaigaTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: TaigaApiParams{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_USER_STORY_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiUserStory struct { + Id uint64 `json:"id"` + Ref int `json:"ref"` + Subject string `json:"subject"` + Status uint64 `json:"status"` + StatusExtraInfo struct { + Name string `json:"name"` + } `json:"status_extra_info"` + IsClosed bool `json:"is_closed"` + CreatedDate *time.Time `json:"created_date"` + ModifiedDate *time.Time `json:"modified_date"` + FinishDate *time.Time `json:"finish_date"` + AssignedTo *uint64 `json:"assigned_to"` + AssignedToExtraInfo *struct { + FullNameDisplay string `json:"full_name_display"` + } `json:"assigned_to_extra_info"` + TotalPoints *float64 `json:"total_points"` + MilestoneId *uint64 `json:"milestone"` + MilestoneName *string `json:"milestone_name"` + Priority *int `json:"priority"` + IsBlocked bool `json:"is_blocked"` + BlockedNote string `json:"blocked_note"` + } + err := json.Unmarshal(row.Data, &apiUserStory) + if err != nil { + return nil, errors.Default.Wrap(err, "error unmarshalling user story") + } + + var assignedTo uint64 + var assignedToName string + if apiUserStory.AssignedTo != nil { + assignedTo = *apiUserStory.AssignedTo + } + if apiUserStory.AssignedToExtraInfo != nil { + assignedToName = apiUserStory.AssignedToExtraInfo.FullNameDisplay + } + var totalPoints float64 + if apiUserStory.TotalPoints != nil { + totalPoints = *apiUserStory.TotalPoints + } + var milestoneId uint64 + if apiUserStory.MilestoneId != nil { + milestoneId = *apiUserStory.MilestoneId + } + var milestoneName string + if apiUserStory.MilestoneName != nil { + milestoneName = *apiUserStory.MilestoneName + } + var priority int + if apiUserStory.Priority != nil { + priority = *apiUserStory.Priority + } + + userStory := &models.TaigaUserStory{ + ConnectionId: data.Options.ConnectionId, + ProjectId: data.Options.ProjectId, + UserStoryId: apiUserStory.Id, + Ref: apiUserStory.Ref, + Subject: apiUserStory.Subject, + Status: apiUserStory.StatusExtraInfo.Name, + IsClosed: apiUserStory.IsClosed, + CreatedDate: apiUserStory.CreatedDate, + ModifiedDate: apiUserStory.ModifiedDate, + FinishedDate: apiUserStory.FinishDate, + AssignedTo: assignedTo, + AssignedToName: assignedToName, + TotalPoints: totalPoints, + MilestoneId: milestoneId, + MilestoneName: milestoneName, + Priority: priority, + IsBlocked: apiUserStory.IsBlocked, + BlockedNote: apiUserStory.BlockedNote, + } + + return []interface{}{userStory}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tapd/api/blueprint_v200.go b/backend/plugins/tapd/api/blueprint_v200.go index 676ed30cf00..abb8695a72b 100644 --- a/backend/plugins/tapd/api/blueprint_v200.go +++ b/backend/plugins/tapd/api/blueprint_v200.go @@ -96,7 +96,7 @@ func makeScopesV200( // get workspace and scope config from db tapdWorkspace, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig - // add wrokspace to scopes + // add workspace to scopes if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { id := idgen.Generate(connection.ID, tapdWorkspace.Id) board := ticket.NewBoard(id, tapdWorkspace.Name) diff --git a/backend/plugins/tapd/models/migrationscripts/archived/tapd_connection.go b/backend/plugins/tapd/models/migrationscripts/archived/tapd_connection.go index b07d89d6a9f..9af60b102c2 100644 --- a/backend/plugins/tapd/models/migrationscripts/archived/tapd_connection.go +++ b/backend/plugins/tapd/models/migrationscripts/archived/tapd_connection.go @@ -39,7 +39,7 @@ type RestConnection struct { BaseConnection `mapstructure:",squash"` Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` Proxy string `mapstructure:"proxy" json:"proxy"` - RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimit"` } type TapdConnection struct { diff --git a/backend/plugins/tempo/api/blueprint_v200.go b/backend/plugins/tempo/api/blueprint_v200.go new file mode 100644 index 00000000000..f786dd65453 --- /dev/null +++ b/backend/plugins/tempo/api/blueprint_v200.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + // Load connection, scope and scopeConfig from the db + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + // Needed for the connection to populate its access tokens + _, err = api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + return plan, scopes, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + + scope := scopeDetail.Scope + // Construct task options for Tempo + task, err := api.MakePipelinePlanTask( + "tempo", + subtaskMetas, + nil, // No entities to select for Tempo + map[string]interface{}{ + "connectionId": scope.ConnectionId, + "teamId": scope.TeamId, + }, + ) + if err != nil { + return nil, err + } + + stage = append(stage, task) + plan[i] = stage + } + + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.TempoTeam, models.TempoScopeConfig], + connection *models.TempoConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0) + for _, scopeDetail := range scopeDetails { + tempoTeam := scopeDetail.Scope + // Add team to scopes + domainTeam := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: didgen.NewDomainIdGenerator(&models.TempoTeam{}).Generate(tempoTeam.ConnectionId, tempoTeam.TeamId), + }, + Name: tempoTeam.Name, + Url: "", // Tempo doesn't provide a direct URL for teams + Type: "team", + } + scopes = append(scopes, domainTeam) + } + return scopes, nil +} diff --git a/backend/plugins/tempo/api/connection.go b/backend/plugins/tempo/api/connection.go new file mode 100644 index 00000000000..5ee6f99059e --- /dev/null +++ b/backend/plugins/tempo/api/connection.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type TempoTestConnResponse struct { + shared.ApiBody + Connection *models.TempoConnection +} + +func testConnection(ctx context.Context, connection models.TempoConnection) (*TempoTestConnResponse, errors.Error) { + // Create API client + apiClient, err := api.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + + // Test connection by fetching teams + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to test connection to Tempo API") + } + + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("failed to connect to Tempo API") + } + + // Sanitize and return response + connection = connection.Sanitize() + body := TempoTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + + return &body, nil +} + +// TestConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // Decode + var connection models.TempoConnection + if err := api.Decode(input.Body, &connection, nil); err != nil { + return nil, err + } + // Test connection + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test tempo connection +// @Summary test tempo connection +// @Description Test Tempo Connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} TempoTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.Convert(err) + } + // Test connection + if result, err := testConnection(context.TODO(), *connection); err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } else { + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil + } +} + +// PostConnections create tempo connection +// @Summary create tempo connection +// @Description Create Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch tempo connection +// @Summary patch tempo connection +// @Description Patch Tempo connection +// @Tags plugins/tempo +// @Param body body models.TempoConnection true "json body" +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a tempo connection +// @Summary delete a tempo connection +// @Description Delete a Tempo connection +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all tempo connections +// @Summary get all tempo connections +// @Description Get all Tempo connections +// @Tags plugins/tempo +// @Success 200 {object} []models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get tempo connection detail +// @Summary get tempo connection detail +// @Description Get Tempo connection detail +// @Tags plugins/tempo +// @Success 200 {object} models.TempoConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/tempo/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} + +// GetTeams get teams for a connection +// @Summary get teams +// @Description Get teams for a Tempo connection +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/teams [GET] +func GetTeams(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.FindByPk(input) + if err != nil { + return nil, err + } + + // Create API client + apiClient, err := api.NewApiClientFromConnection(context.TODO(), basicRes, connection) + if err != nil { + return nil, err + } + + // Get teams from Tempo API + res, err := apiClient.Get("teams", nil, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var teams []models.TempoTeamResponse + err = api.UnmarshalResponse(res, &teams) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + // Convert to tool layer models + result := make([]models.TempoTeam, 0, len(teams)) + for _, t := range teams { + result = append(result, *t.ConvertToToolLayer(connection.ID)) + } + + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} diff --git a/backend/plugins/tempo/api/init.go b/backend/plugins/tempo/api/init.go new file mode 100644 index 00000000000..47ce280c2fe --- /dev/null +++ b/backend/plugins/tempo/api/init.go @@ -0,0 +1,52 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.TempoConnection, models.TempoTeam, models.TempoScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.TempoConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination] +var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + dsHelper = api.NewDataSourceHelper[ + models.TempoConnection, + models.TempoTeam, + models.TempoScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.TempoConnection) models.TempoConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.TempoConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.TempoConnection, models.TempoTeam, TempoRemotePagination](raProxy, listTempoRemoteTeams) + raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.TempoConnection, models.TempoTeam](raProxy, searchTempoRemoteTeams) +} diff --git a/backend/plugins/tempo/api/remote.go b/backend/plugins/tempo/api/remote.go new file mode 100644 index 00000000000..8dd7199939c --- /dev/null +++ b/backend/plugins/tempo/api/remote.go @@ -0,0 +1,196 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/url" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type TempoRemotePagination struct { + Limit int `json:"limit" mapstructure:"limit"` + Offset int `json:"offset" mapstructure:"offset"` +} + +func listTempoRemoteTeams( + connection *models.TempoConnection, + apiClient plugin.ApiClient, + groupId string, + page TempoRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + nextPage *TempoRemotePagination, + err errors.Error, +) { + if page.Limit == 0 { + page.Limit = 50 + } + + queryParams := url.Values{ + "offset": {strconv.Itoa(page.Offset)}, + "limit": {strconv.Itoa(page.Limit)}, + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(connection.ID), + }) + } + + if page.Offset+page.Limit < response.Metadata.Total { + nextPage = &TempoRemotePagination{ + Limit: page.Limit, + Offset: page.Offset + page.Limit, + } + } + + return children, nextPage, nil +} + +// RemoteScopes list all available scopes on the remote server +// @Summary list all available scopes on the remote server +// @Description list all available scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +func searchTempoRemoteTeams( + apiClient plugin.ApiClient, + params *dsmodels.DsRemoteApiScopeSearchParams, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam], + err errors.Error, +) { + var queryParams url.Values + if params.Search != "" { + queryParams = url.Values{ + "name": {params.Search}, + } + } + + res, err := apiClient.Get("teams", queryParams, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to get teams from Tempo API") + } + + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Total int `json:"total"` + } `json:"metadata"` + Results []models.TempoTeamResponse `json:"results"` + } + err = api.UnmarshalResponse(res, &response) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to unmarshal teams response") + } + + for _, team := range response.Results { + if params.Search == "" || strings.Contains(strings.ToLower(team.Name), strings.ToLower(params.Search)) { + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.TempoTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: strconv.FormatInt(team.Id, 10), + ParentId: nil, + Name: team.Name, + FullName: team.Name, + Data: team.ConvertToToolLayer(0), // connectionId overridden by PutMultiple handler + }) + } + } + + start := (params.Page - 1) * params.PageSize + end := start + params.PageSize + if start >= len(children) { + return nil, nil + } + if end > len(children) { + end = len(children) + } + + return children[start:end], nil +} + +// SearchRemoteScopes searches scopes on the remote server +// @Summary searches scopes on the remote server +// @Description searches scopes on the remote server +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TempoTeam] "the parentIds are always null" +// @Tags plugins/tempo +// @Router /plugins/tempo/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeSearch.Get(input) +} + +// Proxy forward API requests to Tempo API +// @Summary Remote server API proxy +// @Description Forward API requests to the specified remote server +// @Param connectionId path int true "connection ID" +// @Param path path string true "path to a API endpoint" +// @Router /plugins/tempo/connections/{connectionId}/proxy/{path} [GET] +// @Tags plugins/tempo +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/tempo/api/scope.go b/backend/plugins/tempo/api/scope.go new file mode 100644 index 00000000000..4f546127117 --- /dev/null +++ b/backend/plugins/tempo/api/scope.go @@ -0,0 +1,120 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.TempoTeam] +type ScopeDetail api.ScopeDetail[models.TempoTeam, models.TempoScopeConfig] + +// PutScope create or update tempo team +// @Summary create or update tempo team +// @Description Create or update Tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes [PUT] +func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// UpdateScope patch to tempo team +// @Summary patch to tempo team +// @Description patch to tempo team +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Param scope body models.TempoTeam true "json" +// @Success 200 {object} models.TempoTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [PATCH] +func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get Tempo teams +// @Summary get Tempo teams +// @Description get Tempo teams +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one Tempo team +// @Summary get one Tempo team +// @Description get one Tempo team +// @Tags plugins/tempo +// @Param connectionId path int false "connection ID" +// @Param scopeId path int false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState get one Tempo team's latest sync state +// @Summary get one Tempo team's latest sync state +// @Description get one Tempo team's latest sync state +// @Tags plugins/tempo +// @Param connectionId path int true "connection ID" +// @Param scopeId path int true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/tempo/api/scope_config.go b/backend/plugins/tempo/api/scope_config.go new file mode 100644 index 00000000000..851aeac8bef --- /dev/null +++ b/backend/plugins/tempo/api/scope_config.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// CreateScopeConfig create scope config for Tempo +// @Summary create scope config for Tempo +// @Description create scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [POST] +func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// UpdateScopeConfig update scope config for Tempo +// @Summary update scope config for Tempo +// @Description update scope config for Tempo +// @Tags plugins/tempo +// @Accept application/json +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Param scopeConfig body models.TempoScopeConfig true "scope config" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [PATCH] +func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {object} models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/tempo +// @Param connectionId path int true "connectionId" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.TempoScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/tempo +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tempo/connections/{connectionId}/scope-configs/{id} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} diff --git a/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv new file mode 100644 index 00000000000..3f552d96c6f --- /dev/null +++ b/backend/plugins/tempo/e2e/raw_tables/_raw_tempo_api_teams.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":0}","{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/17"",""id"":17,""name"":""Abstract BE Team"",""summary"":"""",""key"":""""}","2024-01-15 10:00:00.000" +2,"{""ConnectionId"":1,""TeamId"":0}","{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}","https://api.tempo.io/4/teams","{""self"":""https://api.tempo.io/4/teams/71"",""id"":71,""name"":""SmartShopper (RTL+WHS)"",""summary"":""Microservice Applications"",""key"":""""}","2024-01-15 10:00:00.000" diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv new file mode 100644 index 00000000000..87279da4cc7 --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_teams.csv @@ -0,0 +1,3 @@ +"connection_id","team_id","name","summary","key","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark" +1,17,"Abstract BE Team","","","{""ConnectionId"":1,""TeamId"":0}","_raw_tempo_api_teams",1, +1,71,"SmartShopper (RTL+WHS)","Microservice Applications","","{""ConnectionId"":1,""TeamId"":0}","_raw_tempo_api_teams",2, diff --git a/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv new file mode 100644 index 00000000000..282ba5f5cf6 --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/_tool_tempo_worklogs.csv @@ -0,0 +1,3 @@ +"connection_id","tempo_worklog_id","issue_id","time_spent_seconds","billable_seconds","start_date","start_time","description","author_account_id","created_at","updated_at" +1,12345,10001,3600,3600,"2024-01-15","09:00:00","Working on task","abc123","2024-01-15T10:00:00Z","2024-01-15T10:00:00Z" +1,12346,10002,7200,0,"2024-01-16","14:00:00","Internal meeting","abc123","2024-01-16T16:00:00Z","2024-01-16T16:00:00Z" diff --git a/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv new file mode 100644 index 00000000000..a100e9640dd --- /dev/null +++ b/backend/plugins/tempo/e2e/snapshot_tables/issue_worklogs.csv @@ -0,0 +1,3 @@ +id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +tempo:TempoWorklog:12345,abc123,Working on task,60,2024-01-15T10:00:00.000+00:00,2024-01-15T09:00:00.000+00:00,jira:JiraIssues:1:10001,,,0, +tempo:TempoWorklog:12346,abc123,Internal meeting,120,2024-01-16T16:00:00.000+00:00,2024-01-16T14:00:00.000+00:00,jira:JiraIssues:1:10002,,,0, diff --git a/backend/plugins/tempo/e2e/worklog_test.go b/backend/plugins/tempo/e2e/worklog_test.go new file mode 100644 index 00000000000..bb4ea262bf5 --- /dev/null +++ b/backend/plugins/tempo/e2e/worklog_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +func TestWorklogDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_tempo_worklogs.csv", &models.TempoWorklog{}) + dataflowTester.FlushTabler(&ticket.IssueWorklog{}) + dataflowTester.Subtask(tasks.ConvertWorklogsMeta, taskData) + dataflowTester.VerifyTable( + ticket.IssueWorklog{}, + "./snapshot_tables/issue_worklogs.csv", + []string{"id", "author_id", "time_spent_minutes", "issue_id", "started_date", "logged_date"}, + ) +} + +func TestTeamDataFlow(t *testing.T) { + var plugin impl.Tempo + dataflowTester := e2ehelper.NewDataFlowTester(t, "tempo", plugin) + + taskData := &tasks.TempoTaskData{ + Options: &tasks.TempoOptions{ + ConnectionId: 1, + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_tempo_api_teams.csv", "_raw_tempo_api_teams") + dataflowTester.FlushTabler(&models.TempoTeam{}) + dataflowTester.Subtask(tasks.ExtractTeamsMeta, taskData) + dataflowTester.VerifyTable( + models.TempoTeam{}, + "./snapshot_tables/_tool_tempo_teams.csv", + e2ehelper.ColumnWithRawData( + "connection_id", + "team_id", + "name", + "summary", + "key", + ), + ) +} diff --git a/backend/plugins/tempo/impl/impl.go b/backend/plugins/tempo/impl/impl.go new file mode 100644 index 00000000000..0666033a7fb --- /dev/null +++ b/backend/plugins/tempo/impl/impl.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/tempo/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginModel + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 + plugin.CloseablePluginTask + plugin.PluginSource +} = (*Tempo)(nil) + +type Tempo struct { +} + +func (p Tempo) Connection() dal.Tabler { + return &models.TempoConnection{} +} + +func (p Tempo) Scope() plugin.ToolLayerScope { + return &models.TempoTeam{} +} + +func (p Tempo) ScopeConfig() dal.Tabler { + return &models.TempoScopeConfig{} +} + +func (p Tempo) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Tempo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.TempoConnection{}, + &models.TempoTeam{}, + &models.TempoWorklog{}, + &models.TempoScopeConfig{}, + } +} + +func (p Tempo) Description() string { + return "Collect worklogs from Jira Tempo" +} + +func (p Tempo) Name() string { + return "tempo" +} + +func (p Tempo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectTeamsMeta, + tasks.ExtractTeamsMeta, + tasks.CollectWorklogsMeta, + tasks.ExtractWorklogsMeta, + tasks.ConvertWorklogsMeta, + } +} + +func (p Tempo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.TempoOptions + var err errors.Error + + logger := taskCtx.GetLogger() + logger.Debug("%v", options) + + err = helper.Decode(options, &op, nil) + if err != nil { + return nil, errors.Default.Wrap(err, "could not decode Tempo options") + } + + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("tempo connectionId is invalid") + } + + connection := &models.TempoConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + + err = connectionHelper.FirstById(connection, op.ConnectionId) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to get Tempo connection") + } + + tempoApiClient, err := tasks.NewTempoApiClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to create Tempo api client") + } + + taskData := &tasks.TempoTaskData{ + Options: &op, + ApiClient: tempoApiClient, + Connection: connection, + } + + return taskData, nil +} + +func (p Tempo) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (pp coreModels.PipelinePlan, sc []plugin.Scope, err errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Tempo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/tempo" +} + +func (p Tempo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Tempo) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/proxy/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/teams": { + "GET": api.GetTeams, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.UpdateScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScope, + }, + "connections/:connectionId/scope-configs": { + "POST": api.CreateScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.UpdateScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + } +} + +func (p Tempo) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.TempoTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + data.ApiClient.Release() + return nil +} diff --git a/backend/plugins/tempo/models/connection.go b/backend/plugins/tempo/models/connection.go new file mode 100644 index 00000000000..2322dc991a3 --- /dev/null +++ b/backend/plugins/tempo/models/connection.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/common" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// TempoConn holds the essential information to connect to the Tempo API +type TempoConn struct { + helper.RestConnection `mapstructure:",squash"` + helper.AccessToken `mapstructure:",squash"` +} + +// TempoConnection holds TempoConn plus ID/Name for database storage +type TempoConnection struct { + helper.BaseConnection `mapstructure:",squash"` + TempoConn `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +func (connection TempoConnection) Connection() dal.Tabler { + return &connection +} + +func (connection TempoConnection) Sanitize() TempoConnection { + connection.TempoConn.Token = "" + return connection +} + +// This object conforms to what the frontend currently expects. +type TempoResponse struct { + Name string `json:"name"` + ID uint64 `json:"id"` + TempoConnection +} + +// TempoScopeConfig holds the configuration for a Tempo scope +type TempoScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +func (c TempoScopeConfig) ScopeConfigId() uint64 { + return c.ID +} + +func (c TempoScopeConfig) ScopeConfigConnectionId() uint64 { + return c.ConnectionId +} diff --git a/backend/plugins/tempo/models/migrationscripts/archived/models.go b/backend/plugins/tempo/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..76f0a9edd8d --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/archived/models.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type TempoConnection struct { + archived.BaseConnection `mapstructure:",squash"` + archived.RestConnection `mapstructure:",squash"` + archived.AccessToken `mapstructure:",squash"` +} + +func (TempoConnection) TableName() string { + return "_tool_tempo_connections" +} + +type TempoScopeConfig struct { + archived.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + Name string `gorm:"type:varchar(255)" mapstructure:"name" json:"name"` +} + +func (TempoScopeConfig) TableName() string { + return "_tool_tempo_scope_configs" +} + +type TempoTeam struct { + ConnectionId uint64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + ScopeConfigId uint64 + Key string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Summary string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +type TempoWorklog struct { + ConnectionId uint64 `gorm:"primaryKey"` + TempoWorklogId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"index"` + IssueId int64 `gorm:"index"` + IssueKey string `gorm:"type:varchar(255)"` + AuthorAccountId string `gorm:"type:varchar(255)"` + TimeSpentSeconds int + BillableSeconds int + StartDate string `gorm:"type:varchar(255)"` + StartTime string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + CreatedAt string `gorm:"type:varchar(255)"` + UpdatedAt string `gorm:"type:varchar(255)"` + archived.NoPKModel +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/models/migrationscripts/register.go b/backend/plugins/tempo/models/migrationscripts/register.go new file mode 100644 index 00000000000..42e280a58b9 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All returns all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(tempoInitTables20240401), + } +} diff --git a/backend/plugins/tempo/models/migrationscripts/register_tables.go b/backend/plugins/tempo/models/migrationscripts/register_tables.go new file mode 100644 index 00000000000..ee4e7c8ddb3 --- /dev/null +++ b/backend/plugins/tempo/models/migrationscripts/register_tables.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/tempo/models/migrationscripts/archived" +) + +type tempoInitTables20240401 struct{} + +func (script *tempoInitTables20240401) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.TempoConnection{}, + &archived.TempoScopeConfig{}, + &archived.TempoTeam{}, + &archived.TempoWorklog{}, + ) +} + +func (*tempoInitTables20240401) Version() uint64 { + return 20240401143000 +} + +func (*tempoInitTables20240401) Name() string { + return "Tempo init schemas" +} diff --git a/backend/plugins/tempo/models/team.go b/backend/plugins/tempo/models/team.go new file mode 100644 index 00000000000..d29f3fd72de --- /dev/null +++ b/backend/plugins/tempo/models/team.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "gorm.io/gorm" +) + +var _ plugin.ToolLayerScope = (*TempoTeam)(nil) + +// TempoTeam represents a team in Jira Tempo +type TempoTeam struct { + common.Scope `mapstructure:",squash" gorm:"embedded"` + TeamId int64 `json:"teamId" mapstructure:"teamId" validate:"required" gorm:"primaryKey"` + Id int64 `json:"id" gorm:"-" mapstructure:"-"` // JS scope selector compatibility (mirrors TeamId) + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Summary string `json:"summary" mapstructure:"summary" gorm:"type:varchar(255)"` +} + +// AfterFind populates the virtual Id field after reading from DB +func (t *TempoTeam) AfterFind(_ *gorm.DB) error { + t.Id = t.TeamId + return nil +} + +func (t TempoTeam) ScopeId() string { + return fmt.Sprintf("%d", t.TeamId) +} + +func (t TempoTeam) ScopeName() string { + return t.Name +} + +func (t TempoTeam) ScopeFullName() string { + return fmt.Sprintf("%s - %s", t.Key, t.Name) +} + +func (t TempoTeam) ScopeParams() interface{} { + return &TempoApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (TempoTeam) TableName() string { + return "_tool_tempo_teams" +} + +// TempoApiParams holds the API parameters for Tempo teams +type TempoApiParams struct { + ConnectionId uint64 + TeamId int64 +} + +// TempoTeamResponse represents the API response for a team from Tempo API +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` +} + +// ConvertToToolLayer converts the API response to the tool layer model +func (r TempoTeamResponse) ConvertToToolLayer(connectionId uint64) *TempoTeam { + return &TempoTeam{ + Scope: common.Scope{ + ConnectionId: connectionId, + }, + TeamId: r.Id, + Id: r.Id, + Key: r.Key, + Name: r.Name, + Summary: r.Summary, + } +} diff --git a/backend/plugins/tempo/models/worklog.go b/backend/plugins/tempo/models/worklog.go new file mode 100644 index 00000000000..780c69ee597 --- /dev/null +++ b/backend/plugins/tempo/models/worklog.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type TempoWorklog struct { + common.NoPKModel `mapstructure:",squash" gorm:"embedded"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId" gorm:"primaryKey"` + TempoWorklogId int64 `json:"tempoWorklogId" mapstructure:"tempoWorklogId" gorm:"primaryKey"` + TeamId int64 `json:"teamId" mapstructure:"teamId" gorm:"index"` + IssueId int64 `json:"issueId" mapstructure:"issueId" gorm:"index"` + IssueKey string `json:"issueKey" mapstructure:"issueKey" gorm:"type:varchar(255)"` + AuthorAccountId string `json:"authorAccountId" mapstructure:"authorAccountId" gorm:"type:varchar(255)"` + TimeSpentSeconds int `json:"timeSpentSeconds" mapstructure:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds" mapstructure:"billableSeconds"` + StartDate string `json:"startDate" mapstructure:"startDate" gorm:"type:varchar(255)"` + StartTime string `json:"startTime" mapstructure:"startTime" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description" gorm:"type:text"` + CreatedAt string `json:"createdAt" mapstructure:"createdAt" gorm:"type:varchar(255)"` + UpdatedAt string `json:"updatedAt" mapstructure:"updatedAt" gorm:"type:varchar(255)"` +} + +func (TempoWorklog) TableName() string { + return "_tool_tempo_worklogs" +} diff --git a/backend/plugins/tempo/tasks/api_client.go b/backend/plugins/tempo/tasks/api_client.go new file mode 100644 index 00000000000..8dcefb13472 --- /dev/null +++ b/backend/plugins/tempo/tasks/api_client.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + BaseURL = "https://api.tempo.io/4" +) + +// NewTempoApiClient creates a new Tempo API client +func NewTempoApiClient( + taskCtx plugin.TaskContext, + connection *models.TempoConnection, +) (*helper.ApiAsyncClient, errors.Error) { + apiClient, err := helper.NewApiClientFromConnection(taskCtx.GetContext(), taskCtx, connection) + if err != nil { + return nil, err + } + return helper.CreateAsyncApiClient(taskCtx, apiClient, nil) +} diff --git a/backend/plugins/tempo/tasks/tasks.go b/backend/plugins/tempo/tasks/tasks.go new file mode 100644 index 00000000000..547282d6a5a --- /dev/null +++ b/backend/plugins/tempo/tasks/tasks.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +const ( + RAW_WORKLOG_TABLE = "tempo_api_worklogs" + RAW_TEAM_TABLE = "tempo_api_teams" +) + +// TempoOptions holds the options for the Tempo plugin +type TempoOptions struct { + ConnectionId uint64 `mapstructure:"connectionId" json:"connectionId"` + ScopeConfigId uint64 `mapstructure:"scopeConfigId" json:"scopeConfigId"` + ScopeConfig *models.TempoScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"` + TeamId int64 `mapstructure:"teamId" json:"teamId"` + FromDate string `mapstructure:"fromDate" json:"fromDate"` + ToDate string `mapstructure:"toDate" json:"toDate"` +} + +// TempoTaskData holds the data for a Tempo task +type TempoTaskData struct { + Options *TempoOptions + ApiClient *helper.ApiAsyncClient + Connection *models.TempoConnection +} diff --git a/backend/plugins/tempo/tasks/team_collector.go b/backend/plugins/tempo/tasks/team_collector.go new file mode 100644 index 00000000000..1ac9e2f5bf5 --- /dev/null +++ b/backend/plugins/tempo/tasks/team_collector.go @@ -0,0 +1,115 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectTeamsMeta = plugin.SubTaskMeta{ + Name: "collect-teams", + Description: "Collect teams from Tempo API", + EntryPoint: CollectTeams, + EnabledByDefault: true, + Dependencies: nil, + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }) + if err != nil { + return err + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: "teams", + PageSize: 50, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 50} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/team_extractor.go b/backend/plugins/tempo/tasks/team_extractor.go new file mode 100644 index 00000000000..1ecbc345c4b --- /dev/null +++ b/backend/plugins/tempo/tasks/team_extractor.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractTeamsMeta = plugin.SubTaskMeta{ + Name: "extract_teams", + EntryPoint: ExtractTeams, + EnabledByDefault: true, + Description: "Extract teams from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// API response for Tempo team +type TempoTeamResponse struct { + Id int64 `json:"id"` + Key string `json:"key"` + Name string `json:"name"` + Summary string `json:"summary"` + Self string `json:"self"` +} + +func ExtractTeams(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_TEAM_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiTeam TempoTeamResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiTeam)) + if err != nil { + return nil, err + } + + // Transform to tool layer model + team := &models.TempoTeam{ + TeamId: apiTeam.Id, + Key: apiTeam.Key, + Name: apiTeam.Name, + Summary: apiTeam.Summary, + } + team.ConnectionId = data.Options.ConnectionId + + return []interface{}{team}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_collector.go b/backend/plugins/tempo/tasks/worklog_collector.go new file mode 100644 index 00000000000..28738e35240 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_collector.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var CollectWorklogsMeta = plugin.SubTaskMeta{ + Name: "collect_worklogs", + EntryPoint: CollectWorklogs, + EnabledByDefault: true, + Description: "Collect worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }) + if err != nil { + return err + } + + urlTemplate := "worklogs" + if data.Options.TeamId != 0 { + urlTemplate = fmt.Sprintf("worklogs/team/%d", data.Options.TeamId) + } + + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + ApiClient: data.ApiClient, + UrlTemplate: urlTemplate, + PageSize: 1000, + GetTotalPages: func(res *http.Response, args *api.ApiCollectorArgs) (int, errors.Error) { + var response struct { + Metadata struct { + Count int `json:"count"` + Limit int `json:"limit"` + Total int `json:"total"` + Offset int `json:"offset"` + } `json:"metadata"` + } + if err := api.UnmarshalResponse(res, &response); err != nil { + return 0, err + } + totalPages := (response.Metadata.Total + args.PageSize - 1) / args.PageSize + return totalPages, nil + }, + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + query := url.Values{} + pager := reqData.Pager + if pager == nil { + pager = &api.Pager{Page: 1, Skip: 0, Size: 1000} + } + query.Set("offset", strconv.Itoa(pager.Skip)) + query.Set("limit", strconv.Itoa(pager.Size)) + + if data.Options.TeamId != 0 { + fromDate := data.Options.FromDate + toDate := data.Options.ToDate + if fromDate == "" { + since := time.Now().AddDate(0, 0, -90) + fromDate = since.Format("2006-01-02") + } + if toDate == "" { + toDate = time.Now().Format("2006-01-02") + } + query.Set("from", fromDate) + query.Set("to", toDate) + } else { + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + since := apiCollector.GetSince() + query.Set("updatedFrom", since.Format(time.RFC3339)) + } + } + + return query, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var response struct { + Results []json.RawMessage `json:"results"` + } + err := api.UnmarshalResponse(res, &response) + if err != nil { + return nil, err + } + return response.Results, nil + }, + }) + + if err != nil { + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/tempo/tasks/worklog_convertor.go b/backend/plugins/tempo/tasks/worklog_convertor.go new file mode 100644 index 00000000000..60ee32a1416 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_convertor.go @@ -0,0 +1,155 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ConvertWorklogsMeta = plugin.SubTaskMeta{ + Name: "convert_worklogs", + EntryPoint: ConvertWorklogs, + EnabledByDefault: true, + Description: "Convert worklogs to domain layer", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + connectionId := data.Options.ConnectionId + + logger.Info("converting Tempo worklogs to domain layer") + + issueIdMapping, err := buildIssueIdMapping(db, connectionId) + if err != nil { + return errors.Default.Wrap(err, "failed to build issue ID mapping") + } + + clauses := []dal.Clause{ + dal.Select("*"), + dal.From("_tool_tempo_worklogs"), + dal.Where("connection_id = ?", connectionId), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return errors.Default.Wrap(err, "failed to query Tempo worklogs") + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: connectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + InputRowType: reflect.TypeOf(models.TempoWorklog{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + tempoWorklog := inputRow.(*models.TempoWorklog) + + domainIssueId := "" + if domainId, ok := issueIdMapping[tempoWorklog.IssueId]; ok { + domainIssueId = domainId + } else { + domainIssueId = fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, tempoWorklog.IssueId) + } + + domainWorklogId := fmt.Sprintf("tempo:TempoWorklog:%d", tempoWorklog.TempoWorklogId) + timeSpentMinutes := tempoWorklog.TimeSpentSeconds / 60 + + var startedDate *time.Time + if tempoWorklog.StartDate != "" && tempoWorklog.StartTime != "" { + if t, err := time.Parse("2006-01-02 15:04:05", tempoWorklog.StartDate+" "+tempoWorklog.StartTime); err == nil { + startedDate = &t + } + } + + var loggedDate *time.Time + if tempoWorklog.CreatedAt != "" { + if t, err := time.Parse(time.RFC3339, tempoWorklog.CreatedAt); err == nil { + loggedDate = &t + } + } + + worklog := &ticket.IssueWorklog{ + DomainEntity: domainlayer.DomainEntity{ + Id: domainWorklogId, + }, + IssueId: domainIssueId, + AuthorId: tempoWorklog.AuthorAccountId, + TimeSpentMinutes: timeSpentMinutes, + StartedDate: startedDate, + LoggedDate: loggedDate, + Comment: tempoWorklog.Description, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return errors.Default.Wrap(err, "failed to create data converter") + } + + return converter.Execute() +} + +func buildIssueIdMapping(db dal.Dal, connectionId uint64) (map[int64]string, errors.Error) { + mapping := make(map[int64]string) + + if !db.HasTable("_tool_jira_issues") { + return mapping, nil + } + + clauses := []dal.Clause{ + dal.Select("issue_id"), + dal.From("_tool_jira_issues"), + dal.Where("connection_id = ?", connectionId), + } + + rows, err := db.Cursor(clauses...) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query jira issues") + } + defer rows.Close() + + for rows.Next() { + var issueId uint64 + if err := rows.Scan(&issueId); err != nil { + return nil, errors.Default.Wrap(err, "failed to scan issue") + } + domainId := fmt.Sprintf("jira:JiraIssues:%d:%d", connectionId, issueId) + mapping[int64(issueId)] = domainId + } + + return mapping, nil +} diff --git a/backend/plugins/tempo/tasks/worklog_extractor.go b/backend/plugins/tempo/tasks/worklog_extractor.go new file mode 100644 index 00000000000..31370868d38 --- /dev/null +++ b/backend/plugins/tempo/tasks/worklog_extractor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/tempo/models" +) + +var ExtractWorklogsMeta = plugin.SubTaskMeta{ + Name: "extract_worklogs", + EntryPoint: ExtractWorklogs, + EnabledByDefault: true, + Description: "Extract worklogs from Tempo API", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type TempoWorklogAuthor struct { + AccountId string `json:"accountId"` +} + +type TempoWorklogResponse struct { + TempoWorklogId int64 `json:"tempoWorklogId"` + Issue struct { + Id int64 `json:"id"` + } `json:"issue"` + TimeSpentSeconds int `json:"timeSpentSeconds"` + BillableSeconds int `json:"billableSeconds"` + StartDate string `json:"startDate"` + StartTime string `json:"startTime"` + Description string `json:"description"` + Author TempoWorklogAuthor `json:"author"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +func ExtractWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*TempoTaskData) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: models.TempoApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Table: RAW_WORKLOG_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var apiWorklog TempoWorklogResponse + err := errors.Convert(json.Unmarshal(row.Data, &apiWorklog)) + if err != nil { + return nil, err + } + + worklog := &models.TempoWorklog{ + ConnectionId: data.Options.ConnectionId, + TempoWorklogId: apiWorklog.TempoWorklogId, + IssueId: apiWorklog.Issue.Id, + TimeSpentSeconds: apiWorklog.TimeSpentSeconds, + BillableSeconds: apiWorklog.BillableSeconds, + StartDate: apiWorklog.StartDate, + StartTime: apiWorklog.StartTime, + Description: apiWorklog.Description, + AuthorAccountId: apiWorklog.Author.AccountId, + CreatedAt: apiWorklog.CreatedAt, + UpdatedAt: apiWorklog.UpdatedAt, + } + + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/tempo/tempo.go b/backend/plugins/tempo/tempo.go new file mode 100644 index 00000000000..8b75aee5e4d --- /dev/null +++ b/backend/plugins/tempo/tempo.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/tempo/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Tempo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "tempo"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "tempo connection id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go b/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go index 643c48c8841..b13d1233cb0 100644 --- a/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go +++ b/backend/plugins/testmo/models/migrationscripts/20250629_add_scope_config_id_to_projects.go @@ -21,13 +21,21 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" ) type addScopeConfigIdToProjects struct{} +type TestmoProject20250629 struct { + ScopeConfigId uint64 `gorm:"type:BIGINT NOT NULL DEFAULT 0"` +} + +func (TestmoProject20250629) TableName() string { + return "_tool_testmo_projects" +} + func (*addScopeConfigIdToProjects) Up(basicRes context.BasicRes) errors.Error { - db := basicRes.GetDal() - return db.Exec("ALTER TABLE _tool_testmo_projects ADD COLUMN scope_config_id bigint NOT NULL DEFAULT 0") + return migrationhelper.AutoMigrateTables(basicRes, &TestmoProject20250629{}) } func (*addScopeConfigIdToProjects) Version() uint64 { diff --git a/backend/plugins/webhook/api/deployments.go b/backend/plugins/webhook/api/deployments.go index fc2a463ef62..b65ca9d0873 100644 --- a/backend/plugins/webhook/api/deployments.go +++ b/backend/plugins/webhook/api/deployments.go @@ -126,7 +126,7 @@ func PostDeploymentsByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceO // @Router /projects/:projectName/deployments [POST] func PostDeploymentsByProjectName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { // find or create the connection for this project - connection, err, shouldReturn := getOrCreateConnection(input) + connection, err, shouldReturn := getOrCreateConnection(input, "deployments") if shouldReturn { return nil, err } @@ -134,10 +134,10 @@ func PostDeploymentsByProjectName(input *plugin.ApiResourceInput) (*plugin.ApiRe return postDeployments(input, connection, err) } -func getOrCreateConnection(input *plugin.ApiResourceInput) (*models.WebhookConnection, errors.Error, bool) { +func getOrCreateConnection(input *plugin.ApiResourceInput, webhookSuffix string) (*models.WebhookConnection, errors.Error, bool) { connection := &models.WebhookConnection{} projectName := input.Params["projectName"] - webhookName := fmt.Sprintf("%s_deployments", projectName) + webhookName := fmt.Sprintf("%s_%s", projectName, webhookSuffix) err := findByProjectName(connection, input.Params, pluginName, webhookName) dal := basicRes.GetDal() if err != nil { diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index 49ea72b51f1..e280d4c9f8e 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -19,11 +19,12 @@ package api import ( "fmt" - "github.com/apache/incubator-devlake/core/log" - "github.com/apache/incubator-devlake/helpers/dbhelper" "net/http" "time" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/helpers/dbhelper" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -112,6 +113,26 @@ func PostIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, return postIssue(input, err, connection) } +// PostIssuesByProjectName +// @Summary create issue by project name +// @Description Create issue by project name. The webhook connection will be created automatically if it does not exist.
+// @Description example: {"url":"","issue_key":"DLK-1234","title":"a feature from DLK","description":"","epic_key":"","type":"BUG","status":"TODO","original_status":"created","story_point":0,"resolution_date":null,"created_date":"2020-01-01T12:00:00+00:00","updated_date":null,"lead_time_minutes":0,"parent_issue_key":"DLK-1200","priority":"","original_estimate_minutes":0,"time_spent_minutes":0,"time_remaining_minutes":0,"creator_id":"user1131","creator_name":"Nick name 1","assignee_id":"user1132","assignee_name":"Nick name 2","severity":"","component":""} +// @Tags plugins/webhook +// @Param body body WebhookIssueRequest true "json body" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 403 {string} errcode.Error "Forbidden" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /projects/:projectName/issues [POST] +func PostIssuesByProjectName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + // find or create the connection for this project + connection, err, shouldReturn := getOrCreateConnection(input, "issues") + if shouldReturn { + return nil, err + } + return postIssue(input, err, connection) +} + func postIssue(input *plugin.ApiResourceInput, err errors.Error, connection *models.WebhookConnection) (*plugin.ApiResourceOutput, errors.Error) { if err != nil { return nil, err @@ -234,6 +255,42 @@ func CloseIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro return closeIssue(input, err, connection) } +// CloseIssueByBodyRequest is the body for the body-based close endpoint +type CloseIssueByBodyRequest struct { + IssueKey string `mapstructure:"issueKey" validate:"required,max=255"` + ResolutionDate *time.Time `mapstructure:"resolutionDate"` + OriginalStatus string `mapstructure:"originalStatus"` +} + +// CloseIssueByBody +// @Summary close an issue (body-based) +// @Description Close an incident by passing issueKey in the request body. +// @Description Use this when the client (e.g. Kibana) cannot construct a dynamic URL. +// @Tags plugins/webhook +// @Param connectionId path int true "connection ID" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/{connectionId}/issue/close [POST] +func CloseIssueByBody(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} + // CloseIssueByName // @Summary set issue's status to DONE // @Description set issue's status to DONE @@ -248,6 +305,34 @@ func CloseIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput return closeIssue(input, err, connection) } +// CloseIssueByBodyByName +// @Summary close an issue by connection name (body-based) +// @Description Close an incident using connection name + issueKey in request body. +// @Tags plugins/webhook +// @Param connectionName path string true "connection name" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/by-name/{connectionName}/issue/close [POST] +func CloseIssueByBodyByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.FirstByName(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} + func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *models.WebhookConnection) (*plugin.ApiResourceOutput, errors.Error) { if err != nil { return nil, err @@ -265,6 +350,16 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo } domainIssue.Status = ticket.DONE domainIssue.OriginalStatus = `` + now := time.Now() + if domainIssue.ResolutionDate == nil { + domainIssue.ResolutionDate = &now + } + if domainIssue.LeadTimeMinutes == nil || *domainIssue.LeadTimeMinutes == 0 { + if domainIssue.CreatedDate != nil { + temp := uint(domainIssue.ResolutionDate.Sub(*domainIssue.CreatedDate).Minutes()) + domainIssue.LeadTimeMinutes = &temp + } + } // save err = tx.Update(domainIssue) if err != nil { @@ -278,6 +373,15 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo if err == nil { domainIncident.Status = ticket.DONE domainIncident.OriginalStatus = `` + if domainIncident.ResolutionDate == nil { + domainIncident.ResolutionDate = &now + } + if domainIncident.LeadTimeMinutes == nil || *domainIncident.LeadTimeMinutes == 0 { + if domainIncident.CreatedDate != nil { + temp := uint(domainIncident.ResolutionDate.Sub(*domainIncident.CreatedDate).Minutes()) + domainIncident.LeadTimeMinutes = &temp + } + } // save err = tx.Update(domainIncident) if err != nil { diff --git a/backend/plugins/webhook/api/pull_requests.go b/backend/plugins/webhook/api/pull_requests.go index a01bb6c4d93..50fcee5c43a 100644 --- a/backend/plugins/webhook/api/pull_requests.go +++ b/backend/plugins/webhook/api/pull_requests.go @@ -77,7 +77,7 @@ type WebhookPullRequestReq struct { // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 403 {string} errcode.Error "Forbidden" // @Failure 500 {string} errcode.Error "Internal Error" -// @Router /plugins/webhook/connections/:connectionId/pullrequests [POST] +// @Router /plugins/webhook/connections/:connectionId/pull_requests [POST] func PostPullRequests(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.WebhookConnection{} err := connectionHelper.First(connection, input.Params) @@ -96,7 +96,7 @@ func PostPullRequests(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 403 {string} errcode.Error "Forbidden" // @Failure 500 {string} errcode.Error "Internal Error" -// @Router /plugins/webhook/connections/by-name/:connectionName/pullrequests [POST] +// @Router /plugins/webhook/connections/by-name/:connectionName/pull_requests [POST] func PostPullRequestsByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { connection := &models.WebhookConnection{} err := connectionHelper.FirstByName(connection, input.Params) @@ -104,6 +104,27 @@ func PostPullRequestsByName(input *plugin.ApiResourceInput) (*plugin.ApiResource return postPullRequests(input, connection, err) } +// PostPullRequestsByProjectName +// @Summary create pull requests by project name +// @Description Create pull request by project name. The webhook connection will be created automatically if it does not exist.
+// @Description example1: {"id": "pr1","baseRepoId": "webhook:1","headRepoId": "repo_fork1","status": "MERGED","originalStatus": "OPEN","displayTitle": "Feature: Add new functionality","description": "This PR adds new features","url": "https://github.com/org/repo/pull/1","authorName": "johndoe","authorId": "johnd123","mergedByName": "janedoe","mergedById": "janed123","parentPrId": "","pullRequestKey": 1,"createdDate": "2025-02-20T16:17:36Z","mergedDate": "2025-02-20T17:17:36Z","closedDate": null,"type": "feature","component": "backend","mergeCommitSha": "bf0a79c57dff8f5f1f393de315ee5105a535e059","headRef": "repo_fork1:feature-branch","baseRef": "main","baseCommitSha": "e73325c2c9863f42ea25871cbfaeebcb8edcf604","headCommitSha": "b22f772f1197edfafd4cc5fe679a2d299ec12837","additions": 100,"deletions": 50,"isDraft": false}
+// @Description "baseRepoId" should be equal to "webhook:{connectionId}" for consistent DORA calculations +// @Tags plugins/webhook +// @Param body body WebhookPullRequestReq true "json body" +// @Success 200 +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 403 {string} errcode.Error "Forbidden" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /projects/:projectName/pull_requests [POST] +func PostPullRequestsByProjectName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err, shouldReturn := getOrCreateConnection(input, "pull_requests") + if shouldReturn { + return nil, err + } + + return postPullRequests(input, connection, err) +} + func postPullRequests(input *plugin.ApiResourceInput, connection *models.WebhookConnection, err errors.Error) (*plugin.ApiResourceOutput, errors.Error) { if err != nil { return nil, err diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index 9a67683584e..97dc59adce5 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -99,6 +99,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/:connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + "connections/:connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, ":connectionId/deployments": { "POST": api.PostDeployments, }, @@ -111,6 +114,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler ":connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + ":connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, "connections/by-name/:connectionName": { "GET": api.GetConnectionByName, "PATCH": api.PatchConnectionByName, @@ -128,8 +134,17 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/by-name/:connectionName/issue/:issueKey/close": { "POST": api.CloseIssueByName, }, + "connections/by-name/:connectionName/issue/close": { + "POST": api.CloseIssueByBodyByName, + }, "projects/:projectName/deployments": { "POST": api.PostDeploymentsByProjectName, }, + "projects/:projectName/issues": { + "POST": api.PostIssuesByProjectName, + }, + "projects/:projectName/pull_requests": { + "POST": api.PostPullRequestsByProjectName, + }, } } diff --git a/backend/plugins/zentao/models/migrationscripts/archived/connection.go b/backend/plugins/zentao/models/migrationscripts/archived/connection.go index dab806d988b..7bee6db349b 100644 --- a/backend/plugins/zentao/models/migrationscripts/archived/connection.go +++ b/backend/plugins/zentao/models/migrationscripts/archived/connection.go @@ -46,7 +46,7 @@ type RestConnection struct { BaseConnection `mapstructure:",squash"` Endpoint string `mapstructure:"endpoint" validate:"required" json:"endpoint"` Proxy string `mapstructure:"proxy" json:"proxy"` - RateLimitPerHour int `comment:"api request rate limt per hour" json:"rateLimit"` + RateLimitPerHour int `comment:"api request rate limit per hour" json:"rateLimit"` } type BaseConnection struct { diff --git a/backend/plugins/zentao/tasks/execution_summary_dev_extractor.go b/backend/plugins/zentao/tasks/execution_summary_dev_extractor.go index 7d3fd24a38e..b77ab4f2638 100644 --- a/backend/plugins/zentao/tasks/execution_summary_dev_extractor.go +++ b/backend/plugins/zentao/tasks/execution_summary_dev_extractor.go @@ -33,7 +33,7 @@ var ExtractExecutionSummaryDevMeta = plugin.SubTaskMeta{ Name: "extractExecutionSummaryDev", EntryPoint: ExtractExecutionSummaryDev, EnabledByDefault: true, - Description: "extract Zentao execution summary from build-in page api", + Description: "extract Zentao execution summary from built-in page api", DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, } diff --git a/backend/plugins/zentao/tasks/task_collector.go b/backend/plugins/zentao/tasks/task_collector.go index ea8a3976a6e..ac0d9d7e6d9 100644 --- a/backend/plugins/zentao/tasks/task_collector.go +++ b/backend/plugins/zentao/tasks/task_collector.go @@ -88,7 +88,7 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error { // extract task's children childTasks, err := extractChildrenWithDFS(task) if err != nil { - return nil, errors.Default.New(fmt.Sprintf("extract task: %v chidren err: %v", task, err)) + return nil, errors.Default.New(fmt.Sprintf("extract task: %v children err: %v", task, err)) } for _, task := range childTasks { allTaskRecords[task.Id] = task diff --git a/backend/python/DevelopmentSetup.md b/backend/python/DevelopmentSetup.md index f026fe76091..4bc885a96a3 100644 --- a/backend/python/DevelopmentSetup.md +++ b/backend/python/DevelopmentSetup.md @@ -14,7 +14,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> -# For `make e2e-test` to run properly, the following steps must be taken: +# Python prerequisites for `make e2e-test` + +`make e2e-test` also requires a test database and, for some service tests, `libgit2`. See +[`backend/test/Readme.md`](../test/Readme.md) for the full local E2E test flow. + +For the Python plugin tests to run properly, the following steps must be taken: 1. The following packages are required for Ubuntu: `libffi-dev default-libmysqlclient-dev libpq-dev` 2. `python3.9` is required by the time of this document. diff --git a/backend/python/pydevlake/pydevlake/api.py b/backend/python/pydevlake/pydevlake/api.py index a50b71d9424..cc47601cdfd 100644 --- a/backend/python/pydevlake/pydevlake/api.py +++ b/backend/python/pydevlake/pydevlake/api.py @@ -221,7 +221,7 @@ def get_next_page_id(self, response) -> Optional[int | str]: """ Extracts or compute the id of the next page from the response, e.g. incrementing the value of `page` of a JSON body. - This id will be suplied to the next request via `set_next_page_param`. + This id will be supplied to the next request via `set_next_page_param`. Returning None indicates that the response is the last page. """ pass diff --git a/backend/python/pydevlake/pydevlake/domain_layer/code.py b/backend/python/pydevlake/pydevlake/domain_layer/code.py index cffe7f50297..bd28802c907 100644 --- a/backend/python/pydevlake/pydevlake/domain_layer/code.py +++ b/backend/python/pydevlake/pydevlake/domain_layer/code.py @@ -50,7 +50,7 @@ class PullRequest(DomainModel, table=True): class PullRequestLabels(NoPKModel, table=True): __tablename__ = 'pull_request_labels' pull_request_id: str = Field(primary_key=True) - label_name: str + label_name: str = Field(primary_key=True) class PullRequestCommit(NoPKModel, table=True): @@ -94,7 +94,7 @@ class Commit(NoPKModel, table=True): class CommitParent(NoPKModel, table=True): __tablename__ = 'commit_parents' commit_sha: str = Field(primary_key=True) - parent_commit_sha: str + parent_commit_sha: str = Field(primary_key=True) class CommitsDiff(DomainModel, table=True): @@ -112,6 +112,7 @@ class RefCommit(NoPKModel, table=True): new_commit_sha: str old_commit_sha: str + class Component(NoPKModel, table=True): __tablename__ = 'components' repo_id: str diff --git a/backend/server/api/README.md b/backend/server/api/README.md index 887a85b90ff..04273b38526 100644 --- a/backend/server/api/README.md +++ b/backend/server/api/README.md @@ -18,7 +18,7 @@ limitations under the License. ### Summary -Users can set pipepline plan by config-ui to create schedule jobs. +Users can set pipeline plan by config-ui to create schedule jobs. And config-ui will send blueprint request with cronConfig in crontab format. ### Cron Job diff --git a/backend/server/api/api.go b/backend/server/api/api.go index d7a39071d35..b0dfd74dd94 100644 --- a/backend/server/api/api.go +++ b/backend/server/api/api.go @@ -35,6 +35,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/server/api/auth" _ "github.com/apache/incubator-devlake/server/api/docs" "github.com/apache/incubator-devlake/server/api/ping" "github.com/apache/incubator-devlake/server/api/shared" @@ -53,9 +54,9 @@ const DB_MIGRATING = `Database migration is in progress. Please wait until it is var basicRes context.BasicRes func Init() { - // Initialize services services.Init() basicRes = services.GetBasicRes() + auth.Init(basicRes) } func InjectCustomService(pipelineNotifier services.PipelineNotificationService, projectService services.ProjectService) errors.Error { @@ -104,9 +105,16 @@ func CreateApiServer() *gin.Engine { router.GET("/health", ping.Health) router.GET("/version", version.Get) - // Api keys + // Auth chain order matters: REST API key first (its own short-circuit), + // then the push API key gate, then OIDC session, then oauth2-proxy header + // (only sets USER if not yet set and the forwarded secret matches), then the terminal 401 gate, + // finally CSRF on unsafe methods. router.Use(RestAuthentication(router, basicRes)) + router.Use(RequirePushAuthentication(basicRes)) + router.Use(auth.OIDCAuthentication()) router.Use(OAuth2ProxyAuthentication(basicRes)) + router.Use(auth.RequireAuth()) + router.Use(auth.CSRFProtect()) return router } @@ -119,7 +127,8 @@ func SetupApiServer(router *gin.Engine) { router.UseRawPath = true // router.UnescapePathValues = false - // Endpoint to proceed database migration + // Endpoint to proceed database migration — listed in auth.publicPaths because + // auth tables may not exist yet when migration is pending. router.GET("/proceed-db-migration", func(ctx *gin.Context) { // Execute database migration errors.Must(services.ExecuteMigration()) diff --git a/backend/server/api/auth/auth.go b/backend/server/api/auth/auth.go new file mode 100644 index 00000000000..aead556a5bc --- /dev/null +++ b/backend/server/api/auth/auth.go @@ -0,0 +1,536 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package auth implements user-facing OIDC login for DevLake. Supports any +// number of OIDC providers configured via OIDC_PROVIDERS + per-provider env +// vars; routes /auth/login?provider= to the corresponding IdP. +package auth + +import ( + stdctx "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "golang.org/x/oauth2" + + corectx "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/helpers/oidchelper" + "github.com/apache/incubator-devlake/server/api/shared" +) + +// Auth-related route paths, defined in one place so router registration and +// the middleware whitelist cannot drift. +const ( + PathMethods = "/auth/methods" + PathLogin = "/auth/login" + PathCallback = "/auth/callback" + PathLogout = "/auth/logout" + PathUserInfo = "/auth/userinfo" +) + +// lastSeenThrottle bounds DB writes to one per-jti per window. Tracking +// "last activity" doesn't need per-request precision. +const lastSeenThrottle = 5 * time.Minute + +// Service holds everything the auth handlers and middlewares need. It is the +// unit of testability for this package: tests build one with stub deps and +// drive the gin handlers directly. +type Service struct { + cfg *oidchelper.Config + providers map[string]*oidchelper.Provider + logger log.Logger + db dal.Dal + revoked *revocationCache + + lastSeenMu sync.Mutex + lastSeen map[string]time.Time +} + +// defaultService is populated by Init and backs the package-level handler / +// middleware wrappers. Tests that need isolation should construct their own +// *Service via NewService. +var ( + defaultService *Service + initOnce sync.Once +) + +// Init builds the default Service from env config and starts the background +// loops. Panics if auth is enabled but the config is incomplete. +func Init(basicRes corectx.BasicRes) { + initOnce.Do(func() { + s, err := NewService(stdctx.Background(), basicRes) + if err != nil { + panic(fmt.Errorf("auth init: %w", err)) + } + defaultService = s + }) +} + +// NewService is the testable constructor. ctx governs the lifetime of the +// revocation refresher and session cleanup goroutines. +func NewService(ctx stdctx.Context, basicRes corectx.BasicRes) (*Service, error) { + cfg, err := oidchelper.LoadConfig(basicRes) + if err != nil { + return nil, err + } + s := &Service{ + cfg: cfg, + providers: map[string]*oidchelper.Provider{}, + logger: basicRes.GetLogger(), + db: basicRes.GetDal(), + revoked: newRevocationCache(), + lastSeen: map[string]time.Time{}, + } + if cfg.AuthEnabled { + startRefresher(ctx, s.revoked, s.db, s.logger) + startSessionCleanup(ctx, s.db, s.logger) + } + if cfg.OIDCEnabled { + for name, pc := range cfg.Providers { + s.providers[name] = oidchelper.NewProvider(pc) + s.logger.Info("OIDC provider %q enabled (issuer=%s, client=%s)", name, pc.IssuerURL, pc.ClientID) + } + } else if cfg.AuthEnabled { + s.logger.Info("AUTH_ENABLED but OIDC_ENABLED=false: only API-key/proxy auth will work") + } + return s, nil +} + +func (s *Service) Config() *oidchelper.Config { return s.cfg } + +// Config returns the default service's config. Nil before Init has run. +func Config() *oidchelper.Config { + if defaultService == nil { + return nil + } + return defaultService.cfg +} + +type ProviderInfo struct { + Name string `json:"name"` + DisplayName string `json:"displayName"` + LoginURL string `json:"loginUrl"` +} + +type Methods struct { + Providers []ProviderInfo `json:"providers,omitempty"` + APIKey *APIKey `json:"apiKey,omitempty"` +} + +type APIKey struct { + Enabled bool `json:"enabled"` +} + +func GetMethods(c *gin.Context) { defaultService.GetMethods(c) } + +// @Summary List enabled login methods +// @Tags framework/auth +// @Success 200 {object} Methods +// @Router /auth/methods [get] +func (s *Service) GetMethods(c *gin.Context) { + out := Methods{APIKey: &APIKey{Enabled: true}} + if s.cfg != nil && s.cfg.OIDCEnabled { + for _, name := range s.cfg.ProviderNames() { + pc := s.cfg.Providers[name] + out.Providers = append(out.Providers, ProviderInfo{ + Name: name, + DisplayName: pc.DisplayName, + LoginURL: PathLogin + "?provider=" + url.QueryEscape(name), + }) + } + } + shared.ApiOutputSuccess(c, out, http.StatusOK) +} + +func LoginInit(c *gin.Context) { defaultService.LoginInit(c) } + +// @Summary Begin OIDC login +// @Tags framework/auth +// @Param provider query string false "provider name (required when more than one is configured)" +// @Param return_url query string false "where to redirect after login" +// @Success 303 +// @Router /auth/login [get] +func (s *Service) LoginInit(c *gin.Context) { + if !s.ensureOIDC(c) { + return + } + name, p, ok := s.pickProvider(c, c.Query("provider")) + if !ok { + return + } + returnURL := safeReturnURL(c.Query("return_url")) + + verifier, err := newPKCEVerifier() + if err != nil { + fail(c, http.StatusInternalServerError, "pkce generate", err) + return + } + nonce, err := oidchelper.NewNonce() + if err != nil { + fail(c, http.StatusInternalServerError, "state nonce", err) + return + } + encoded, err := oidchelper.EncodeState(s.cfg.SessionSecret, &oidchelper.StatePayload{ + Provider: name, + Nonce: nonce, + ReturnURL: returnURL, + PKCEVerifier: verifier, + IssuedAt: time.Now(), + }) + if err != nil { + fail(c, http.StatusInternalServerError, "state encode", err) + return + } + oidchelper.SetStateCookie(c, s.cfg, encoded) + + oa, err := p.OAuth2Config(c.Request.Context()) + if err != nil { + fail(c, http.StatusBadGateway, "oauth2 config", err) + return + } + authURL := oa.AuthCodeURL(nonce, + oauth2.SetAuthURLParam("code_challenge", pkceChallenge(verifier)), + oauth2.SetAuthURLParam("code_challenge_method", "S256"), + ) + c.Redirect(http.StatusSeeOther, authURL) +} + +func Callback(c *gin.Context) { defaultService.Callback(c) } + +// @Summary OIDC callback +// @Tags framework/auth +// @Success 303 +// @Router /auth/callback [get] +func (s *Service) Callback(c *gin.Context) { + if !s.ensureOIDC(c) { + return + } + + encoded, err := c.Cookie(oidchelper.StateCookieName) + if err != nil || encoded == "" { + fail(c, http.StatusBadRequest, "missing state cookie", err) + return + } + oidchelper.ClearStateCookie(c, s.cfg) + + state, err := oidchelper.DecodeState(s.cfg.SessionSecret, encoded) + if err != nil { + fail(c, http.StatusBadRequest, "state decode", err) + return + } + // Constant-time compare even though the nonce is non-secret. Keeps the + // auth path free of timing-leak audit questions. + if subtle.ConstantTimeCompare([]byte(c.Query("state")), []byte(state.Nonce)) != 1 { + fail(c, http.StatusBadRequest, "state mismatch", nil) + return + } + p, ok := s.providers[state.Provider] + if !ok { + fail(c, http.StatusBadRequest, "unknown provider in state: "+state.Provider, nil) + return + } + code := c.Query("code") + if code == "" { + if msg := c.Query("error"); msg != "" { + fail(c, http.StatusBadRequest, "idp error: "+msg+" "+c.Query("error_description"), nil) + return + } + fail(c, http.StatusBadRequest, "missing authorization code", nil) + return + } + + oa, err := p.OAuth2Config(c.Request.Context()) + if err != nil { + fail(c, http.StatusBadGateway, "oauth2 config", err) + return + } + exchangeOpts := []oauth2.AuthCodeOption{ + oauth2.SetAuthURLParam("code_verifier", state.PKCEVerifier), + } + if pc := s.cfg.Providers[state.Provider]; pc != nil && pc.UseWorkloadIdentity { + assertion, werr := oidchelper.FederatedAssertion() + if werr != nil { + fail(c, http.StatusInternalServerError, "workload identity assertion", werr) + return + } + exchangeOpts = append(exchangeOpts, + oauth2.SetAuthURLParam("client_assertion_type", "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"), + oauth2.SetAuthURLParam("client_assertion", assertion), + ) + } + tok, err := oa.Exchange(c.Request.Context(), code, exchangeOpts...) + if err != nil { + fail(c, http.StatusBadGateway, "code exchange", err) + return + } + rawID, _ := tok.Extra("id_token").(string) + if rawID == "" { + fail(c, http.StatusBadGateway, "id_token missing from token response", nil) + return + } + idTok, err := p.VerifyIDToken(c.Request.Context(), rawID) + if err != nil { + fail(c, http.StatusUnauthorized, "id_token verify", err) + return + } + + sub, email, name, err := extractUser(idTok) + if err != nil { + fail(c, http.StatusBadGateway, "extract claims", err) + return + } + + if !s.cfg.IsUserAllowed(email) { + fail(c, http.StatusForbidden, "user is not allowed", nil) + return + } + + jti := uuid.NewString() + jwt, expiresAt, err := oidchelper.IssueSession(s.cfg, jti, state.Provider, sub, email, name) + if err != nil { + fail(c, http.StatusInternalServerError, "issue session", err) + return + } + now := time.Now() + if dbErr := CreateSession(s.db, &AuthSession{ + Jti: jti, + Sub: sub, + Email: email, + Name: name, + IssuedAt: now, + ExpiresAt: expiresAt, + LastSeenAt: now, + }); dbErr != nil { + fail(c, http.StatusInternalServerError, "persist session", dbErr) + return + } + csrf, err := oidchelper.NewCSRFToken() + if err != nil { + fail(c, http.StatusInternalServerError, "csrf token", err) + return + } + oidchelper.SetSessionCookie(c, s.cfg, jwt) + oidchelper.SetCSRFCookie(c, s.cfg, csrf) + s.logger.Info("oidc login: provider=%s sub=%s email=%s jti=%s", state.Provider, sub, email, jti) + + c.Redirect(http.StatusSeeOther, state.ReturnURL) +} + +type logoutResponse struct { + OK bool `json:"ok"` + LogoutURL string `json:"logoutUrl,omitempty"` +} + +func Logout(c *gin.Context) { defaultService.Logout(c) } + +// @Summary Logout +// @Tags framework/auth +// @Success 200 {object} logoutResponse +// @Router /auth/logout [post] +func (s *Service) Logout(c *gin.Context) { + if s.cfg == nil || !s.cfg.AuthEnabled { + shared.ApiOutputSuccess(c, logoutResponse{OK: true}, http.StatusOK) + return + } + + var sessionProvider string + if len(s.cfg.SessionSecret) > 0 { + if raw, err := c.Cookie(oidchelper.SessionCookieName); err == nil && raw != "" { + if claims, err := oidchelper.ParseSession(s.cfg.SessionSecret, raw); err == nil && claims.ID != "" { + sessionProvider = claims.Provider + if err := RevokeSession(s.db, claims.ID); err != nil { + s.logger.Error(err, "auth: revoke session row") + } + s.revoked.Add(claims.ID) + } + } + } + oidchelper.ClearSessionCookie(c, s.cfg) + oidchelper.ClearCSRFCookie(c, s.cfg) + + out := logoutResponse{OK: true} + if s.cfg.OIDCEnabled && s.cfg.LogoutRedirect && sessionProvider != "" { + if p, ok := s.providers[sessionProvider]; ok { + if u, err := p.EndSessionURL(c.Request.Context()); err == nil && u != "" { + out.LogoutURL = u + } + } + } + shared.ApiOutputSuccess(c, out, http.StatusOK) +} + +type userInfoResponse struct { + Authenticated bool `json:"authenticated"` + Name string `json:"name"` + Email string `json:"email"` +} + +func UserInfo(c *gin.Context) { defaultService.UserInfo(c) } + +// UserInfo always returns 200 so calling it does not trigger the UI's +// 401-redirect loop; the `authenticated` field discriminates. +// +// @Summary Current user +// @Tags framework/auth +// @Success 200 {object} userInfoResponse +// @Router /auth/userinfo [get] +func (s *Service) UserInfo(c *gin.Context) { + u, ok := shared.GetUser(c) + if !ok || u == nil { + shared.ApiOutputSuccess(c, userInfoResponse{Authenticated: false}, http.StatusOK) + return + } + shared.ApiOutputSuccess(c, userInfoResponse{ + Authenticated: true, + Name: u.Name, + Email: u.Email, + }, http.StatusOK) +} + +// pickProvider resolves the requested provider name. Empty names are allowed +// only when exactly one provider is configured (single-IdP convenience). +func (s *Service) pickProvider(c *gin.Context, requested string) (string, *oidchelper.Provider, bool) { + name := strings.ToLower(strings.TrimSpace(requested)) + if name == "" { + if len(s.providers) == 1 { + for n := range s.providers { + name = n + } + } else { + fail(c, http.StatusBadRequest, "provider query param required (multiple configured)", nil) + return "", nil, false + } + } + p, ok := s.providers[name] + if !ok { + fail(c, http.StatusBadRequest, "unknown provider: "+name, nil) + return "", nil, false + } + return name, p, true +} + +func (s *Service) ensureOIDC(c *gin.Context) bool { + if s.cfg == nil || !s.cfg.OIDCEnabled || len(s.providers) == 0 { + shared.ApiOutputError(c, errors.HttpStatus(http.StatusServiceUnavailable).New("OIDC is not enabled")) + return false + } + return true +} + +// bumpLastSeen records that jti was used. The DB write happens at most once +// per lastSeenThrottle per jti, off the request path. +func (s *Service) bumpLastSeen(jti string) { + now := time.Now() + s.lastSeenMu.Lock() + if last, ok := s.lastSeen[jti]; ok && now.Sub(last) < lastSeenThrottle { + s.lastSeenMu.Unlock() + return + } + s.lastSeen[jti] = now + s.lastSeenMu.Unlock() + + go func() { + if err := UpdateLastSeen(s.db, jti, now); err != nil { + s.logger.Debug("auth: bump last_seen jti=%s: %v", jti, err) + } + }() +} + +func fail(c *gin.Context, status int, msg string, cause error) { + wrapped := errors.HttpStatus(status).New(msg) + if cause != nil { + wrapped = errors.HttpStatus(status).Wrap(cause, msg) + } + shared.ApiOutputError(c, wrapped) +} + +// safeReturnURL prevents open-redirect by stripping anything that doesn't +// look like a same-origin path. The // and /\ prefixes get a fast path +// because some browsers normalize backslash → slash and would treat +// /\evil.com as a protocol-relative URL. +func safeReturnURL(raw string) string { + if raw == "" { + return "/" + } + if strings.HasPrefix(raw, "//") || strings.HasPrefix(raw, `/\`) { + return "/" + } + u, err := url.Parse(raw) + if err != nil { + return "/" + } + if u.IsAbs() || u.Host != "" || !strings.HasPrefix(u.Path, "/") { + return "/" + } + return u.RequestURI() +} + +// extractUser pulls user identity out of the verified ID token. Email is +// taken strictly from the `email` claim; we never coerce a username into +// the email column. Display name falls back to preferred_username, then +// email, so the UI always has *something* to render. +func extractUser(tok *oidc.IDToken) (sub, email, name string, err error) { + var claims struct { + Sub string `json:"sub"` + Email string `json:"email"` + PreferredUsername string `json:"preferred_username"` + Name string `json:"name"` + } + if err := tok.Claims(&claims); err != nil { + return "", "", "", err + } + if claims.Sub == "" { + return "", "", "", fmt.Errorf("id_token missing sub claim") + } + name = claims.Name + if name == "" { + name = claims.PreferredUsername + } + if name == "" { + name = claims.Email + } + return claims.Sub, claims.Email, name, nil +} + +func pkceChallenge(verifier string) string { + sum := sha256.Sum256([]byte(verifier)) + return base64.RawURLEncoding.EncodeToString(sum[:]) +} + +// newPKCEVerifier returns a 64-char random URL-safe string per RFC 7636. +func newPKCEVerifier() (string, error) { + b := make([]byte, 48) + if _, err := io.ReadFull(rand.Reader, b); err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(b), nil +} diff --git a/backend/server/api/auth/auth_test.go b/backend/server/api/auth/auth_test.go new file mode 100644 index 00000000000..7235b91442b --- /dev/null +++ b/backend/server/api/auth/auth_test.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import "testing" + +func TestSafeReturnURL(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {"empty", "", "/"}, + {"root", "/", "/"}, + {"normal path", "/projects", "/projects"}, + {"path with query", "/projects?tab=2", "/projects?tab=2"}, + {"path with fragment kept", "/projects#section", "/projects"}, + {"absolute http URL", "http://evil.com/x", "/"}, + {"absolute https URL", "https://evil.com/x", "/"}, + {"protocol-relative", "//evil.com", "/"}, + {"protocol-relative with path", "//evil.com/x", "/"}, + {"backslash variant", `/\evil.com`, "/"}, + {"backslash variant with path", `/\evil.com/x`, "/"}, + {"missing leading slash", "projects", "/"}, + {"javascript scheme", "javascript:alert(1)", "/"}, + {"data scheme", "data:text/html,x", "/"}, + {"unparsable", "://", "/"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := safeReturnURL(tc.in); got != tc.want { + t.Errorf("safeReturnURL(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} diff --git a/backend/server/api/auth/cleanup.go b/backend/server/api/auth/cleanup.go new file mode 100644 index 00000000000..3dcaf7421e6 --- /dev/null +++ b/backend/server/api/auth/cleanup.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/log" +) + +const ( + sessionCleanupInterval = 24 * time.Hour + // Keep expired rows around as a soft audit trail before pruning. 30d + // covers any reasonable "who logged in last month" question without + // letting the table grow unbounded. + sessionRetentionAfterExpiry = 30 * 24 * time.Hour +) + +// startSessionCleanup deletes long-expired session rows on a daily timer. +// First sweep runs after one interval, not at boot, so deploys don't pay +// the cleanup cost during the noisy startup window. +func startSessionCleanup(ctx context.Context, db dal.Dal, logger log.Logger) { + go func() { + t := time.NewTicker(sessionCleanupInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + cutoff := time.Now().Add(-sessionRetentionAfterExpiry) + if err := db.Delete(&AuthSession{}, dal.Where("expires_at < ?", cutoff)); err != nil { + logger.Error(err, "auth: session cleanup") + continue + } + logger.Info("auth: pruned auth_sessions rows expired before %s", cutoff.Format(time.RFC3339)) + } + } + }() +} diff --git a/backend/server/api/auth/handlers_test.go b/backend/server/api/auth/handlers_test.go new file mode 100644 index 00000000000..7067fa97876 --- /dev/null +++ b/backend/server/api/auth/handlers_test.go @@ -0,0 +1,426 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "math/big" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/mock" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/oidchelper" + "github.com/apache/incubator-devlake/impls/logruslog" + mockdal "github.com/apache/incubator-devlake/mocks/core/dal" +) + +// fakeIdP is a minimal OIDC provider sufficient for go-oidc discovery, +// token exchange, and JWKS verification. +type fakeIdP struct { + server *httptest.Server + key *rsa.PrivateKey + keyID string + issuer string + + mu sync.Mutex + lastCode string + subject string + email string + name string +} + +func newFakeIdP(t *testing.T) *fakeIdP { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("rsa.GenerateKey: %v", err) + } + idp := &fakeIdP{ + key: key, + keyID: "test-key", + subject: "user-123", + email: "alice@example.com", + name: "Alice", + } + mux := http.NewServeMux() + mux.HandleFunc("/.well-known/openid-configuration", idp.handleDiscovery) + mux.HandleFunc("/jwks", idp.handleJWKS) + mux.HandleFunc("/token", idp.handleToken) + mux.HandleFunc("/authorize", func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "authorize endpoint not exercised by these tests", http.StatusNotImplemented) + }) + mux.HandleFunc("/logout", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + }) + idp.server = httptest.NewServer(mux) + idp.issuer = idp.server.URL + t.Cleanup(idp.server.Close) + return idp +} + +func (f *fakeIdP) handleDiscovery(w http.ResponseWriter, _ *http.Request) { + doc := map[string]any{ + "issuer": f.issuer, + "authorization_endpoint": f.issuer + "/authorize", + "token_endpoint": f.issuer + "/token", + "jwks_uri": f.issuer + "/jwks", + "end_session_endpoint": f.issuer + "/logout", + "response_types_supported": []string{"code"}, + "subject_types_supported": []string{"public"}, + "id_token_signing_alg_values_supported": []string{"RS256"}, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(doc) +} + +func (f *fakeIdP) handleJWKS(w http.ResponseWriter, _ *http.Request) { + pub := f.key.PublicKey + doc := map[string]any{ + "keys": []map[string]string{{ + "kty": "RSA", + "use": "sig", + "kid": f.keyID, + "alg": "RS256", + "n": base64.RawURLEncoding.EncodeToString(pub.N.Bytes()), + "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(pub.E)).Bytes()), + }}, + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(doc) +} + +func (f *fakeIdP) handleToken(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + code := r.FormValue("code") + f.mu.Lock() + f.lastCode = code + f.mu.Unlock() + + // oauth2.Config.Exchange sends client credentials via HTTP Basic by + // default. Fall back to the form for clients that send them there. + clientID := r.FormValue("client_id") + if u, _, ok := r.BasicAuth(); ok { + clientID = u + } + + tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{ + "iss": f.issuer, + "aud": clientID, + "sub": f.subject, + "email": f.email, + "name": f.name, + "iat": time.Now().Unix(), + "exp": time.Now().Add(5 * time.Minute).Unix(), + }) + tok.Header["kid"] = f.keyID + signed, err := tok.SignedString(f.key) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "access_token": "dummy-access", + "token_type": "Bearer", + "expires_in": 300, + "id_token": signed, + }) +} + +// newTestService builds a Service backed by the fake IdP and a permissive +// dal mock that accepts the auth-store writes the handlers issue. +func newTestService(t *testing.T, idp *fakeIdP) (*Service, *mockdal.Dal) { + t.Helper() + pc := &oidchelper.ProviderConfig{ + Name: "test", + IssuerURL: idp.issuer, + ClientID: "test-client", + ClientSecret: "test-secret", + RedirectURL: "http://localhost/auth/callback", + Scopes: []string{"openid", "profile", "email"}, + DisplayName: "Test IdP", + } + cfg := &oidchelper.Config{ + AuthEnabled: true, + OIDCEnabled: true, + Providers: map[string]*oidchelper.ProviderConfig{"test": pc}, + LogoutRedirect: true, + SessionSecret: []byte("test-secret-with-at-least-32-bytes!"), + SessionTTL: time.Hour, + CookieSecure: false, + } + db := &mockdal.Dal{} + db.On("Create", mock.Anything, mock.Anything).Return(nil) + db.On("UpdateColumn", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + db.On("All", mock.Anything, mock.Anything).Return(nil) + db.On("Delete", mock.Anything, mock.Anything).Return(nil) + + s := &Service{ + cfg: cfg, + providers: map[string]*oidchelper.Provider{"test": oidchelper.NewProvider(pc)}, + logger: logruslog.Global, + db: db, + revoked: newRevocationCache(), + lastSeen: map[string]time.Time{}, + } + return s, db +} + +func newTestRouter(s *Service) *gin.Engine { + gin.SetMode(gin.TestMode) + r := gin.New() + r.Use(s.OIDCAuthentication()) + r.Use(s.RequireAuth()) + r.Use(s.CSRFProtect()) + r.GET(PathLogin, s.LoginInit) + r.GET(PathCallback, s.Callback) + r.GET(PathUserInfo, s.UserInfo) + r.POST(PathLogout, s.Logout) + return r +} + +// extractCookie pulls a Set-Cookie value by name from a recorded response. +func extractCookie(t *testing.T, resp *http.Response, name string) *http.Cookie { + t.Helper() + for _, c := range resp.Cookies() { + if c.Name == name { + return c + } + } + t.Fatalf("cookie %q not found in response", name) + return nil +} + +// stateNonceFromCookie decodes the encrypted state cookie back into the +// nonce, which the test then echoes as the OIDC `state` query parameter. +func stateNonceFromCookie(t *testing.T, secret []byte, cookieValue string) string { + t.Helper() + state, err := oidchelper.DecodeState(secret, cookieValue) + if err != nil { + t.Fatalf("decode state cookie: %v", err) + } + return state.Nonce +} + +func TestLoginInitSetsStateCookieAndRedirects(t *testing.T) { + idp := newFakeIdP(t) + s, _ := newTestService(t, idp) + r := newTestRouter(s) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, PathLogin+"?provider=test", nil) + r.ServeHTTP(w, req) + + if w.Code != http.StatusSeeOther { + t.Fatalf("expected 303, got %d: %s", w.Code, w.Body.String()) + } + loc := w.Header().Get("Location") + if !strings.Contains(loc, "code_challenge=") || !strings.Contains(loc, "code_challenge_method=S256") { + t.Fatalf("expected PKCE params on auth URL, got %q", loc) + } + resp := w.Result() + defer resp.Body.Close() + if c := extractCookie(t, resp, oidchelper.StateCookieName); c.Value == "" { + t.Fatal("state cookie was empty") + } +} + +func TestFullLoginCallbackFlow(t *testing.T) { + idp := newFakeIdP(t) + s, db := newTestService(t, idp) + r := newTestRouter(s) + + // 1. /auth/login: capture the state cookie. + loginW := httptest.NewRecorder() + r.ServeHTTP(loginW, httptest.NewRequest(http.MethodGet, PathLogin+"?provider=test", nil)) + if loginW.Code != http.StatusSeeOther { + t.Fatalf("login: expected 303, got %d", loginW.Code) + } + stateCookie := extractCookie(t, loginW.Result(), oidchelper.StateCookieName) + nonce := stateNonceFromCookie(t, s.cfg.SessionSecret, stateCookie.Value) + + // 2. /auth/callback: must succeed and set session + csrf cookies. + cbReq := httptest.NewRequest(http.MethodGet, + PathCallback+"?code=fake-code&state="+url.QueryEscape(nonce), nil) + cbReq.AddCookie(stateCookie) + cbW := httptest.NewRecorder() + r.ServeHTTP(cbW, cbReq) + if cbW.Code != http.StatusSeeOther { + t.Fatalf("callback: expected 303, got %d body=%s", cbW.Code, cbW.Body.String()) + } + sessionCookie := extractCookie(t, cbW.Result(), oidchelper.SessionCookieName) + csrfCookie := extractCookie(t, cbW.Result(), oidchelper.CSRFCookieName) + if sessionCookie.Value == "" || csrfCookie.Value == "" { + t.Fatal("expected session and csrf cookies to be set") + } + db.AssertCalled(t, "Create", mock.Anything, mock.Anything) + + // 3. /auth/userinfo with the session cookie should report authenticated. + uiReq := httptest.NewRequest(http.MethodGet, PathUserInfo, nil) + uiReq.AddCookie(sessionCookie) + uiW := httptest.NewRecorder() + r.ServeHTTP(uiW, uiReq) + if uiW.Code != http.StatusOK { + t.Fatalf("userinfo: expected 200, got %d", uiW.Code) + } + var userResp userInfoResponse + if err := json.Unmarshal(uiW.Body.Bytes(), &userResp); err != nil { + t.Fatalf("decode userinfo: %v: %s", err, uiW.Body.String()) + } + if !userResp.Authenticated || userResp.Email != idp.email { + t.Fatalf("unexpected userinfo: %+v body=%s", userResp, uiW.Body.String()) + } + + // 4. /auth/logout with session + csrf header should succeed and revoke. + logoutReq := httptest.NewRequest(http.MethodPost, PathLogout, nil) + logoutReq.AddCookie(sessionCookie) + logoutReq.AddCookie(csrfCookie) + logoutReq.Header.Set(oidchelper.CSRFHeaderName, csrfCookie.Value) + logoutW := httptest.NewRecorder() + r.ServeHTTP(logoutW, logoutReq) + if logoutW.Code != http.StatusOK { + t.Fatalf("logout: expected 200, got %d body=%s", logoutW.Code, logoutW.Body.String()) + } + db.AssertCalled(t, "UpdateColumn", mock.Anything, "revoked_at", mock.Anything, mock.Anything) + + // 5. The same session cookie now belongs to a revoked jti; userinfo + // should report not authenticated. + uiReq2 := httptest.NewRequest(http.MethodGet, PathUserInfo, nil) + uiReq2.AddCookie(sessionCookie) + uiW2 := httptest.NewRecorder() + r.ServeHTTP(uiW2, uiReq2) + var userResp2 userInfoResponse + if err := json.Unmarshal(uiW2.Body.Bytes(), &userResp2); err != nil { + t.Fatalf("decode userinfo (revoked): %v", err) + } + if userResp2.Authenticated { + t.Fatal("expected revoked session to no longer be authenticated") + } +} + +func TestCallbackRejectsStateMismatch(t *testing.T) { + idp := newFakeIdP(t) + s, _ := newTestService(t, idp) + r := newTestRouter(s) + + loginW := httptest.NewRecorder() + r.ServeHTTP(loginW, httptest.NewRequest(http.MethodGet, PathLogin+"?provider=test", nil)) + stateCookie := extractCookie(t, loginW.Result(), oidchelper.StateCookieName) + + cbReq := httptest.NewRequest(http.MethodGet, PathCallback+"?code=x&state=not-the-real-nonce", nil) + cbReq.AddCookie(stateCookie) + cbW := httptest.NewRecorder() + r.ServeHTTP(cbW, cbReq) + if cbW.Code != http.StatusBadRequest { + t.Fatalf("expected 400 on state mismatch, got %d", cbW.Code) + } + if !strings.Contains(cbW.Body.String(), "state mismatch") { + t.Fatalf("expected state mismatch in body, got %s", cbW.Body.String()) + } +} + +func TestCallbackRejectsMissingStateCookie(t *testing.T) { + idp := newFakeIdP(t) + s, _ := newTestService(t, idp) + r := newTestRouter(s) + + cbReq := httptest.NewRequest(http.MethodGet, PathCallback+"?code=x&state=anything", nil) + cbW := httptest.NewRecorder() + r.ServeHTTP(cbW, cbReq) + if cbW.Code != http.StatusBadRequest { + t.Fatalf("expected 400 with missing state cookie, got %d", cbW.Code) + } + if !strings.Contains(cbW.Body.String(), "missing state cookie") { + t.Fatalf("expected missing state cookie in body, got %s", cbW.Body.String()) + } +} + +func TestCSRFRequiredOnUnsafeMethod(t *testing.T) { + idp := newFakeIdP(t) + s, _ := newTestService(t, idp) + r := newTestRouter(s) + r.POST("/some-mutation", func(c *gin.Context) { c.Status(http.StatusNoContent) }) + + // Issue a session via the fake-callback so we have valid cookies. + loginW := httptest.NewRecorder() + r.ServeHTTP(loginW, httptest.NewRequest(http.MethodGet, PathLogin+"?provider=test", nil)) + stateCookie := extractCookie(t, loginW.Result(), oidchelper.StateCookieName) + nonce := stateNonceFromCookie(t, s.cfg.SessionSecret, stateCookie.Value) + + cbReq := httptest.NewRequest(http.MethodGet, + PathCallback+"?code=c&state="+url.QueryEscape(nonce), nil) + cbReq.AddCookie(stateCookie) + cbW := httptest.NewRecorder() + r.ServeHTTP(cbW, cbReq) + sessionCookie := extractCookie(t, cbW.Result(), oidchelper.SessionCookieName) + csrfCookie := extractCookie(t, cbW.Result(), oidchelper.CSRFCookieName) + + // POST without the CSRF header is rejected. + bad := httptest.NewRequest(http.MethodPost, "/some-mutation", nil) + bad.AddCookie(sessionCookie) + bad.AddCookie(csrfCookie) + badW := httptest.NewRecorder() + r.ServeHTTP(badW, bad) + if badW.Code != http.StatusForbidden { + t.Fatalf("expected 403 without csrf header, got %d", badW.Code) + } + + // POST with a matching header passes. + good := httptest.NewRequest(http.MethodPost, "/some-mutation", nil) + good.AddCookie(sessionCookie) + good.AddCookie(csrfCookie) + good.Header.Set(oidchelper.CSRFHeaderName, csrfCookie.Value) + goodW := httptest.NewRecorder() + r.ServeHTTP(goodW, good) + if goodW.Code != http.StatusNoContent { + t.Fatalf("expected 204 with csrf header, got %d body=%s", goodW.Code, goodW.Body.String()) + } +} + +func TestLogoutPublicWhenNoSession(t *testing.T) { + idp := newFakeIdP(t) + s, _ := newTestService(t, idp) + r := newTestRouter(s) + + // No session cookie, no csrf header. publicPaths should let it through + // and the handler should respond 200. + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodPost, PathLogout, nil)) + if w.Code != http.StatusOK { + t.Fatalf("expected logout-without-session to return 200, got %d body=%s", w.Code, w.Body.String()) + } +} + +// Compile-time assertion that the testify mock satisfies the dal.Dal +// interface so the tests catch any new method that gets added. +var _ dal.Dal = (*mockdal.Dal)(nil) diff --git a/backend/server/api/auth/middleware.go b/backend/server/api/auth/middleware.go new file mode 100644 index 00000000000..231f4d5460c --- /dev/null +++ b/backend/server/api/auth/middleware.go @@ -0,0 +1,169 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "crypto/subtle" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/oidchelper" + "github.com/apache/incubator-devlake/server/api/shared" +) + +// publicPaths is the set of routes reachable without authentication. +// /auth/userinfo and /auth/logout are public so the UI can poll identity +// and clear its session even when the cookie has lapsed; both handlers +// short-circuit gracefully when no user is set. +var publicPaths = map[string]struct{}{ + "/ping": {}, + "/ready": {}, + "/health": {}, + "/version": {}, + "/proceed-db-migration": {}, + PathMethods: {}, + PathLogin: {}, + PathCallback: {}, + PathLogout: {}, + PathUserInfo: {}, +} + +func OIDCAuthentication() gin.HandlerFunc { return defaultService.OIDCAuthentication() } + +func RequireAuth() gin.HandlerFunc { return defaultService.RequireAuth() } + +func CSRFProtect() gin.HandlerFunc { return defaultService.CSRFProtect() } + +// OIDCAuthentication reads the session cookie, verifies the JWT, and sets +// common.USER on the context. Soft authenticator: invalid/missing cookies +// pass through and RequireAuth decides whether to reject. +func (s *Service) OIDCAuthentication() gin.HandlerFunc { + return func(c *gin.Context) { + if s.cfg == nil || !s.cfg.AuthEnabled { + c.Next() + return + } + if _, ok := shared.GetUser(c); ok { + c.Next() + return + } + if len(s.cfg.SessionSecret) == 0 { + c.Next() + return + } + raw, err := c.Cookie(oidchelper.SessionCookieName) + if err != nil || raw == "" { + c.Next() + return + } + claims, err := oidchelper.ParseSession(s.cfg.SessionSecret, raw) + if err != nil { + s.logger.Debug("invalid session cookie: %v", err) + oidchelper.ClearSessionCookie(c, s.cfg) + c.Next() + return + } + if s.revoked != nil && s.revoked.IsRevoked(claims.ID) { + s.logger.Debug("revoked session presented: jti=%s", claims.ID) + oidchelper.ClearSessionCookie(c, s.cfg) + c.Next() + return + } + c.Set(common.USER, &common.User{ + Name: claims.Name, + Email: claims.Email, + }) + s.bumpLastSeen(claims.ID) + c.Next() + } +} + +// RequireAuth is the terminal gate. It only becomes a no-op when +// AUTH_ENABLED=false is set explicitly. +func (s *Service) RequireAuth() gin.HandlerFunc { + return func(c *gin.Context) { + if s.cfg == nil || !s.cfg.AuthEnabled { + c.Next() + return + } + if isPublicPath(c.Request.URL.Path) { + c.Next() + return + } + if _, ok := shared.GetUser(c); ok { + c.Next() + return + } + c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{ + "success": false, + "message": "unauthorized", + }) + } +} + +func isPublicPath(path string) bool { + if _, ok := publicPaths[path]; ok { + return true + } + return strings.HasPrefix(path, "/swagger/") +} + +// CSRFProtect rejects unsafe (POST/PUT/DELETE/PATCH) requests authenticated +// via the session cookie unless they echo the CSRF cookie back as the +// X-CSRF-Token header. Other auth methods (API key Bearer, oauth2-proxy +// header) are not subject to CSRF; they don't ride on ambient cookies. +// +// SameSite=Lax already blocks the textbook cross-origin form-POST attack; +// this is defense-in-depth for shared-parent-domain deployments and any +// future GET endpoint that is upgraded to a mutation. +func (s *Service) CSRFProtect() gin.HandlerFunc { + return func(c *gin.Context) { + if s.cfg == nil || !s.cfg.AuthEnabled { + c.Next() + return + } + switch c.Request.Method { + case http.MethodGet, http.MethodHead, http.MethodOptions: + c.Next() + return + } + if isPublicPath(c.Request.URL.Path) { + c.Next() + return + } + // CSRF only applies when the caller is authenticating via the session + // cookie. Bearer tokens and proxy headers aren't replayable cross-site. + if _, err := c.Cookie(oidchelper.SessionCookieName); err != nil { + c.Next() + return + } + cookie, err := c.Cookie(oidchelper.CSRFCookieName) + header := c.GetHeader(oidchelper.CSRFHeaderName) + if err != nil || cookie == "" || header == "" || subtle.ConstantTimeCompare([]byte(cookie), []byte(header)) != 1 { + c.AbortWithStatusJSON(http.StatusForbidden, gin.H{ + "success": false, + "message": "csrf token missing or invalid", + }) + return + } + c.Next() + } +} diff --git a/backend/server/api/auth/revocation_cache.go b/backend/server/api/auth/revocation_cache.go new file mode 100644 index 00000000000..b704d1bb4b9 --- /dev/null +++ b/backend/server/api/auth/revocation_cache.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "context" + "sync" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/log" +) + +// revocationRefreshInterval is how often the cache reloads the revoked-jti +// set from the database. Revocations propagate within this window. +const revocationRefreshInterval = 30 * time.Second + +// revocationCache holds the set of revoked session jtis. The MVP keeps it +// fully in memory because the set is tiny (only sessions that were +// explicitly revoked, pruned on natural expiry) and per-request DB lookups +// would otherwise dominate the auth middleware's hot path. +type revocationCache struct { + mu sync.RWMutex + revoked map[string]struct{} +} + +func newRevocationCache() *revocationCache { + return &revocationCache{revoked: map[string]struct{}{}} +} + +func (r *revocationCache) IsRevoked(jti string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + _, ok := r.revoked[jti] + return ok +} + +func (r *revocationCache) Add(jti string) { + r.mu.Lock() + r.revoked[jti] = struct{}{} + r.mu.Unlock() +} + +// Refresh reloads the revoked set from the DB while preserving any Adds +// that landed concurrently with the DB read. The race we're guarding: +// Logout calls RevokeSession then revoked.Add; if Refresh's DB read happens +// between those two steps, the new map would be missing the jti. Carrying +// forward "additions since the snapshot" plugs that gap. +func (r *revocationCache) Refresh(db dal.Dal) error { + r.mu.RLock() + before := make(map[string]struct{}, len(r.revoked)) + for j := range r.revoked { + before[j] = struct{}{} + } + r.mu.RUnlock() + + jtis, err := ListRevoked(db) + if err != nil { + return err + } + + r.mu.Lock() + defer r.mu.Unlock() + next := make(map[string]struct{}, len(jtis)) + for _, j := range jtis { + next[j] = struct{}{} + } + // Anything in the current set that wasn't in the snapshot was Added + // during the DB read; keep it. Anything in both `before` and the + // current set but absent from the DB result has expired naturally and + // is safe to drop. + for j := range r.revoked { + if _, wasBefore := before[j]; !wasBefore { + next[j] = struct{}{} + } + } + r.revoked = next + return nil +} + +// startRefresher loads the cache once synchronously, then refreshes it on a +// timer. The synchronous first load means we never accept revoked sessions +// in a startup window. +func startRefresher(ctx context.Context, cache *revocationCache, db dal.Dal, logger log.Logger) { + if err := cache.Refresh(db); err != nil { + logger.Error(err, "auth: initial revocation cache load failed") + } + go func() { + t := time.NewTicker(revocationRefreshInterval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if err := cache.Refresh(db); err != nil { + logger.Error(err, "auth: revocation cache refresh failed") + } + } + } + }() +} diff --git a/backend/server/api/auth/revocation_cache_test.go b/backend/server/api/auth/revocation_cache_test.go new file mode 100644 index 00000000000..8b66c2c8d48 --- /dev/null +++ b/backend/server/api/auth/revocation_cache_test.go @@ -0,0 +1,67 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "sync" + "testing" +) + +func TestRevocationCacheEmpty(t *testing.T) { + c := newRevocationCache() + if c.IsRevoked("anything") { + t.Fatal("empty cache should never report revoked") + } +} + +func TestRevocationCacheAdd(t *testing.T) { + c := newRevocationCache() + c.Add("jti-1") + if !c.IsRevoked("jti-1") { + t.Fatal("expected jti-1 to be revoked") + } + if c.IsRevoked("jti-2") { + t.Fatal("jti-2 was never added; should not be revoked") + } +} + +func TestRevocationCacheConcurrentReadsAndWrites(t *testing.T) { + c := newRevocationCache() + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(2) + go func(i int) { + defer wg.Done() + c.Add(jtiKey(i)) + }(i) + go func(i int) { + defer wg.Done() + _ = c.IsRevoked(jtiKey(i)) + }(i) + } + wg.Wait() + for i := 0; i < 100; i++ { + if !c.IsRevoked(jtiKey(i)) { + t.Fatalf("jti %d missing after concurrent adds", i) + } + } +} + +func jtiKey(i int) string { + return "jti-" + string(rune('a'+i%26)) + string(rune('0'+(i/26)%10)) +} diff --git a/backend/server/api/auth/store.go b/backend/server/api/auth/store.go new file mode 100644 index 00000000000..751624e82df --- /dev/null +++ b/backend/server/api/auth/store.go @@ -0,0 +1,76 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" +) + +// AuthSession is the persisted record backing one signed session JWT. It is +// what makes server-side revocation possible: the JWT carries the jti, and a +// row here can be flagged revoked to block reuse before the token's natural +// expiry. +type AuthSession struct { + Jti string `gorm:"primaryKey;type:varchar(36)" json:"jti"` + Sub string `gorm:"type:varchar(255);index" json:"sub"` + Email string `gorm:"type:varchar(255)" json:"email"` + Name string `gorm:"type:varchar(255)" json:"name"` + IssuedAt time.Time `json:"issuedAt"` + ExpiresAt time.Time `gorm:"index" json:"expiresAt"` + RevokedAt *time.Time `gorm:"index" json:"revokedAt,omitempty"` + LastSeenAt time.Time `json:"lastSeenAt"` +} + +func (AuthSession) TableName() string { return "auth_sessions" } + +func CreateSession(db dal.Dal, s *AuthSession) errors.Error { + return db.Create(s) +} + +// RevokeSession marks the session row revoked. No-op if no row matches. +func RevokeSession(db dal.Dal, jti string) errors.Error { + now := time.Now() + return db.UpdateColumn(&AuthSession{}, "revoked_at", now, dal.Where("jti = ?", jti)) +} + +// UpdateLastSeen records that the session was used. Throttled by the caller +// (see Service.bumpLastSeen) so this isn't a per-request DB write. +func UpdateLastSeen(db dal.Dal, jti string, at time.Time) errors.Error { + return db.UpdateColumn(&AuthSession{}, "last_seen_at", at, dal.Where("jti = ?", jti)) +} + +// ListRevoked returns the jti of every still-relevant revoked session, +// excluding ones already past their natural expiry. +func ListRevoked(db dal.Dal) ([]string, errors.Error) { + var rows []AuthSession + err := db.All(&rows, + dal.Select("jti"), + dal.Where("revoked_at IS NOT NULL AND expires_at > ?", time.Now()), + ) + if err != nil { + return nil, err + } + out := make([]string, len(rows)) + for i, r := range rows { + out[i] = r.Jti + } + return out, nil +} diff --git a/backend/server/api/middlewares.go b/backend/server/api/middlewares.go index 86c8c6a6fb7..a988c41288a 100644 --- a/backend/server/api/middlewares.go +++ b/backend/server/api/middlewares.go @@ -18,14 +18,16 @@ limitations under the License. package api import ( + "crypto/subtle" "encoding/base64" "fmt" - "github.com/apache/incubator-devlake/core/log" "net/http" "regexp" "strings" "time" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" @@ -34,12 +36,28 @@ import ( "github.com/gin-gonic/gin" ) -func getOAuthUserInfo(c *gin.Context) (*common.User, error) { +const ( + forwardedUserHeader = "X-Forwarded-User" + forwardedEmailHeader = "X-Forwarded-Email" + forwardedUserSecretHeader = "X-Forwarded-User-Secret" +) + +func getOAuthUserInfo(c *gin.Context, forwardedUserSecret string) (*common.User, error) { if c == nil { return nil, errors.Default.New("request is nil") } - user := c.GetHeader("X-Forwarded-User") - email := c.GetHeader("X-Forwarded-Email") + user := strings.TrimSpace(c.GetHeader(forwardedUserHeader)) + if user == "" { + return nil, nil + } + if forwardedUserSecret == "" { + return nil, errors.Default.New("ignoring forwarded user headers because FORWARDED_USER_SECRET is not configured") + } + providedSecret := strings.TrimSpace(c.GetHeader(forwardedUserSecretHeader)) + if subtle.ConstantTimeCompare([]byte(providedSecret), []byte(forwardedUserSecret)) != 1 { + return nil, errors.Default.New("ignoring forwarded user headers because X-Forwarded-User-Secret did not match") + } + email := strings.TrimSpace(c.GetHeader(forwardedEmailHeader)) return &common.User{ Name: user, Email: email, @@ -74,12 +92,13 @@ func getBasicAuthUserInfo(c *gin.Context, basicRes context.BasicRes) (*common.Us func OAuth2ProxyAuthentication(basicRes context.BasicRes) gin.HandlerFunc { logger := basicRes.GetLogger() + forwardedUserSecret := strings.TrimSpace(basicRes.GetConfigReader().GetString("FORWARDED_USER_SECRET")) return func(c *gin.Context) { _, exist := c.Get(common.USER) if !exist { - user, err := getOAuthUserInfo(c) + user, err := getOAuthUserInfo(c, forwardedUserSecret) if err != nil { - logger.Error(err, "getOAuthUserInfo") + logger.Warn(err, "rejected forwarded user headers") } if user == nil || user.Name == "" { // fetch with basic auth header @@ -113,7 +132,6 @@ func RestAuthentication(router *gin.Engine, basicRes context.BasicRes) gin.Handl path := c.Request.URL.Path // Only open api needs to check api key if !strings.HasPrefix(path, "/rest") { - logger.Debug("path %s will continue", path) c.Next() return } @@ -131,6 +149,53 @@ func RestAuthentication(router *gin.Engine, basicRes context.BasicRes) gin.Handl } } +func RequirePushAuthentication(basicRes context.BasicRes) gin.HandlerFunc { + logger := basicRes.GetLogger() + return func(c *gin.Context) { + path := c.Request.URL.Path + if !strings.HasPrefix(path, "/push/") { + c.Next() + return + } + + authHeader := c.GetHeader("Authorization") + if authHeader == "" { + c.Abort() + c.JSON(http.StatusUnauthorized, &apiBody{ + Success: false, + Message: "token is missing", + }) + return + } + apiKeyStr := strings.TrimPrefix(authHeader, "Bearer ") + if apiKeyStr == authHeader || apiKeyStr == "" { + c.Abort() + c.JSON(http.StatusUnauthorized, &apiBody{ + Success: false, + Message: "token is not present or malformed", + }) + return + } + + db := basicRes.GetDal() + if db == nil { + logger.Error(nil, "db is not initialised") + c.Abort() + c.JSON(http.StatusInternalServerError, &apiBody{ + Success: false, + Message: "database is not initialised", + }) + return + } + + apiKeyHelper := apikeyhelper.NewApiKeyHelper(basicRes, logger) + if !CheckAuthorizationHeader(c, logger, db, apiKeyHelper, authHeader, path) { + return + } + c.Next() + } +} + func CheckAuthorizationHeader(c *gin.Context, logger log.Logger, db dal.Dal, apiKeyHelper *apikeyhelper.ApiKeyHelper, authHeader, path string) bool { if authHeader == "" { c.Abort() diff --git a/backend/server/api/middlewares_forwardsecret_test.go b/backend/server/api/middlewares_forwardsecret_test.go new file mode 100644 index 00000000000..b5e13ed8391 --- /dev/null +++ b/backend/server/api/middlewares_forwardsecret_test.go @@ -0,0 +1,147 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/spf13/viper" + + "github.com/apache/incubator-devlake/core/config" + corecontext "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/apache/incubator-devlake/server/api/shared" +) + +type proxyAuthTestBasicRes struct { + cfg config.ConfigReader + logger log.Logger +} + +func (b *proxyAuthTestBasicRes) GetConfigReader() config.ConfigReader { return b.cfg } +func (b *proxyAuthTestBasicRes) GetConfig(name string) string { return b.cfg.GetString(name) } +func (b *proxyAuthTestBasicRes) GetLogger() log.Logger { return b.logger } +func (b *proxyAuthTestBasicRes) NestedLogger(name string) corecontext.BasicRes { + return &proxyAuthTestBasicRes{cfg: b.cfg, logger: b.logger.Nested(name)} +} +func (b *proxyAuthTestBasicRes) ReplaceLogger(logger log.Logger) corecontext.BasicRes { + return &proxyAuthTestBasicRes{cfg: b.cfg, logger: logger} +} +func (b *proxyAuthTestBasicRes) GetDal() dal.Dal { return nil } + +type proxyAuthResponse struct { + Authenticated bool `json:"authenticated"` + Name string `json:"name"` + Email string `json:"email"` +} + +func newProxyAuthRouter(secret string) *gin.Engine { + gin.SetMode(gin.TestMode) + cfg := viper.New() + cfg.Set("FORWARDED_USER_SECRET", secret) + basicRes := &proxyAuthTestBasicRes{ + cfg: cfg, + logger: logruslog.Global, + } + r := gin.New() + r.Use(OAuth2ProxyAuthentication(basicRes)) + r.GET("/me", func(c *gin.Context) { + user, ok := shared.GetUser(c) + if !ok { + c.JSON(http.StatusOK, proxyAuthResponse{}) + return + } + c.JSON(http.StatusOK, proxyAuthResponse{ + Authenticated: true, + Name: user.Name, + Email: user.Email, + }) + }) + return r +} + +func performProxyAuthRequest(t *testing.T, router *gin.Engine, headers map[string]string) proxyAuthResponse { + t.Helper() + req := httptest.NewRequest(http.MethodGet, "/me", nil) + for key, value := range headers { + req.Header.Set(key, value) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, req) + if recorder.Code != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", recorder.Code, recorder.Body.String()) + } + var body proxyAuthResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &body); err != nil { + t.Fatalf("decode response: %v", err) + } + return body +} + +func TestOAuth2ProxyAuthenticationRejectsUntrustedForwardedHeaders(t *testing.T) { + router := newProxyAuthRouter("shared-secret") + cases := map[string]map[string]string{ + "missing secret header": { + forwardedUserHeader: "admin@example.com", + forwardedEmailHeader: "admin@example.com", + }, + "mismatched secret header": { + forwardedUserHeader: "admin@example.com", + forwardedEmailHeader: "admin@example.com", + forwardedUserSecretHeader: "wrong-secret", + }, + } + for name, headers := range cases { + t.Run(name, func(t *testing.T) { + body := performProxyAuthRequest(t, router, headers) + if body.Authenticated { + t.Fatalf("expected forwarded headers to be rejected, got %+v", body) + } + }) + } +} + +func TestOAuth2ProxyAuthenticationRequiresConfiguredSecret(t *testing.T) { + router := newProxyAuthRouter("") + body := performProxyAuthRequest(t, router, map[string]string{ + forwardedUserHeader: "admin@example.com", + forwardedEmailHeader: "admin@example.com", + forwardedUserSecretHeader: "shared-secret", + }) + if body.Authenticated { + t.Fatalf("expected forwarded headers to be ignored without FORWARDED_USER_SECRET, got %+v", body) + } +} + +func TestOAuth2ProxyAuthenticationAcceptsTrustedForwardedHeaders(t *testing.T) { + router := newProxyAuthRouter("shared-secret") + body := performProxyAuthRequest(t, router, map[string]string{ + forwardedUserHeader: "admin@example.com", + forwardedEmailHeader: "admin@example.com", + forwardedUserSecretHeader: "shared-secret", + }) + if !body.Authenticated || body.Name != "admin@example.com" || body.Email != "admin@example.com" { + t.Fatalf("expected trusted forwarded user, got %+v", body) + } +} diff --git a/backend/server/api/middlewares_test.go b/backend/server/api/middlewares_test.go new file mode 100644 index 00000000000..bb6d1f26824 --- /dev/null +++ b/backend/server/api/middlewares_test.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + corectx "github.com/apache/incubator-devlake/core/context" + contextimpl "github.com/apache/incubator-devlake/impls/context" + "github.com/apache/incubator-devlake/impls/logruslog" + "github.com/gin-gonic/gin" + "github.com/spf13/viper" +) + +func newPushTestBasicRes() corectx.BasicRes { + cfg := viper.New() + cfg.Set("ENCRYPTION_SECRET", strings.Repeat("a", 32)) + return contextimpl.NewDefaultBasicRes(cfg, logruslog.Global, nil) +} + +func TestRequirePushAuthenticationRejectsMissingToken(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(RequirePushAuthentication(newPushTestBasicRes())) + router.POST("/push/:tableName", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "/push/commits", strings.NewReader(`[{}]`)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + if resp.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", resp.Code, http.StatusUnauthorized) + } +} + +func TestRequirePushAuthenticationRejectsMalformedToken(t *testing.T) { + gin.SetMode(gin.TestMode) + router := gin.New() + router.Use(RequirePushAuthentication(newPushTestBasicRes())) + router.POST("/push/:tableName", func(c *gin.Context) { + c.Status(http.StatusOK) + }) + + req := httptest.NewRequest(http.MethodPost, "/push/commits", strings.NewReader(`[{}]`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Basic dGVzdDp0ZXN0") + resp := httptest.NewRecorder() + router.ServeHTTP(resp, req) + + if resp.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want %d", resp.Code, http.StatusUnauthorized) + } +} diff --git a/backend/server/api/push/README.md b/backend/server/api/push/README.md index 999f087b40e..f9aa3cb8e65 100644 --- a/backend/server/api/push/README.md +++ b/backend/server/api/push/README.md @@ -21,6 +21,10 @@ limitations under the License. This is a generic API service that gives our users the ability to inject data directly to their own database using a simple, all-purpose endpoint. +The push API is disabled by default. To enable it safely, you must: +- configure `PUSH_API_ALLOWED_TABLES` with the specific non-internal tables that may be written +- send a Bearer API key whose `allowedPath` matches the `/push/...` endpoint you are calling + ## The Endpoint POST to ```localhost:8080/push/:tableName``` @@ -28,6 +32,9 @@ POST to ```localhost:8080/push/:tableName``` Where "tableName" is the name of the table you wish to insert into For example, "commits" would be ```/push/commits``` +Internal `_devlake_*` tables are never writable through this endpoint, even if +they are listed in `PUSH_API_ALLOWED_TABLES`. + ## The JSON body Include a JSON body that consists of an array of objects you wish to insert. @@ -45,4 +52,3 @@ Please Note: You must know the schema you are inserting into (column names, type ``` - diff --git a/backend/server/api/router.go b/backend/server/api/router.go index 721517cf59a..a26e6b4a454 100644 --- a/backend/server/api/router.go +++ b/backend/server/api/router.go @@ -26,6 +26,7 @@ import ( "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/impls/logruslog" "github.com/apache/incubator-devlake/server/api/apikeys" + "github.com/apache/incubator-devlake/server/api/auth" "github.com/apache/incubator-devlake/server/api/store" "github.com/apache/incubator-devlake/core/plugin" @@ -86,6 +87,13 @@ func RegisterRouter(r *gin.Engine, basicRes context.BasicRes) { r.PUT("/api-keys/:apiKeyId", apikeys.PutApiKey) r.DELETE("/api-keys/:apiKeyId", apikeys.DeleteApiKey) + // auth (OIDC user login) + r.GET(auth.PathMethods, auth.GetMethods) + r.GET(auth.PathLogin, auth.LoginInit) + r.GET(auth.PathCallback, auth.Callback) + r.POST(auth.PathLogout, auth.Logout) + r.GET(auth.PathUserInfo, auth.UserInfo) + // mount all api resources for all plugins resources, err := services.GetPluginsApiResources() if err != nil { diff --git a/backend/server/api/store/store.go b/backend/server/api/store/store.go index 0c7ab3778e5..14e78dc0753 100644 --- a/backend/server/api/store/store.go +++ b/backend/server/api/store/store.go @@ -33,7 +33,7 @@ import ( // @Description Get the value by given key // @Tags framework/projects // @Param storeKey path string true "storeKey" -// @Success 200 {object} json.RawMessage +// @Success 200 {object} interface{} // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 500 {string} errcode.Error "Internal Error" // @Router /store/{storeKey} [get] @@ -53,8 +53,8 @@ func GetStore(c *gin.Context) { // @Tags framework/projects // @Accept application/json // @Param storeKey path string true "storeKey" -// @Param project body json.RawMessage false "json" -// @Success 200 {object} json.RawMessage +// @Param project body interface{} false "json" +// @Success 200 {object} interface{} // @Failure 400 {string} errcode.Error "Bad Request" // @Failure 500 {string} errcode.Error "Internal Error" // @Router /store/{storeKey} [PUT] diff --git a/backend/server/services/blueprint.go b/backend/server/services/blueprint.go index 91bab693420..2fb05c3b718 100644 --- a/backend/server/services/blueprint.go +++ b/backend/server/services/blueprint.go @@ -258,7 +258,7 @@ func DeleteBlueprint(id uint64) errors.Error { var blueprintReloadLock sync.Mutex var bpCronIdMap map[uint64]cron.EntryID -// ReloadBlueprints reloades cronjobs based on blueprints +// ReloadBlueprints reloads cronjobs based on blueprints func ReloadBlueprints() (err errors.Error) { enable := true isManual := false diff --git a/backend/server/services/pipeline.go b/backend/server/services/pipeline.go index d20089a5679..d0cb0dbcb5f 100644 --- a/backend/server/services/pipeline.go +++ b/backend/server/services/pipeline.go @@ -439,7 +439,7 @@ func CancelPipeline(pipelineId uint64) errors.Error { pipeline.Status = models.TASK_CANCELLED err = db.Update(pipeline) if err != nil { - return errors.Default.Wrap(err, "faile to update pipeline") + return errors.Default.Wrap(err, "failed to update pipeline") } // now, with RunPipelineInQueue being block and target pipeline got updated // we should update the related tasks as well @@ -449,7 +449,7 @@ func CancelPipeline(pipelineId uint64) errors.Error { dal.Where("pipeline_id = ?", pipelineId), ) if err != nil { - return errors.Default.Wrap(err, "faile to update pipeline tasks") + return errors.Default.Wrap(err, "failed to update pipeline tasks") } // the target pipeline is pending, no running, no need to perform the actual cancel operation return nil @@ -607,7 +607,7 @@ func RerunPipeline(pipelineId uint64, task *models.Task) (tasks []*models.Task, rerunTasks = append(rerunTasks, rerunTask) } - // mark pipline rerun + // mark pipeline rerun err = tx.UpdateColumn(&models.Pipeline{}, "status", models.TASK_RERUN, dal.Where("id = ?", pipelineId), diff --git a/backend/server/services/project.go b/backend/server/services/project.go index c030a9af53d..a1b92b03ae4 100644 --- a/backend/server/services/project.go +++ b/backend/server/services/project.go @@ -106,7 +106,7 @@ func CreateProject(projectInput *models.ApiInputProject) (*models.ApiOutputProje return nil, err } - // create transaction to updte multiple tables + // create transaction to update multiple tables var err errors.Error tx := db.Begin() defer func() { diff --git a/backend/server/services/pushapi.go b/backend/server/services/pushapi.go index d99d4e9067d..7882b0aa1a4 100644 --- a/backend/server/services/pushapi.go +++ b/backend/server/services/pushapi.go @@ -18,30 +18,15 @@ limitations under the License. package services import ( - "regexp" - "strings" - "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/server/services/pushapiaccess" ) // InsertRow FIXME ... func InsertRow(table string, rows []map[string]interface{}) (int64, errors.Error) { - if !regexp.MustCompile(`^[a-zA-Z0-9_]+$`).MatchString(table) { - return 0, errors.BadInput.New("table name invalid") - } - - if allowedTables := cfg.GetString("PUSH_API_ALLOWED_TABLES"); allowedTables != "" { - allow := false - for _, t := range strings.Split(allowedTables, ",") { - if strings.TrimSpace(t) == table { - allow = true - break - } - } - if !allow { - return 0, errors.Forbidden.New("table name is not in the allowed list") - } + if err := pushapiaccess.ValidateTable(table, cfg.GetString("PUSH_API_ALLOWED_TABLES")); err != nil { + return 0, err } err := db.Create(rows, dal.From(table)) diff --git a/backend/server/services/pushapiaccess/access.go b/backend/server/services/pushapiaccess/access.go new file mode 100644 index 00000000000..40f213be127 --- /dev/null +++ b/backend/server/services/pushapiaccess/access.go @@ -0,0 +1,54 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pushapiaccess + +import ( + "regexp" + "strings" + + "github.com/apache/incubator-devlake/core/errors" +) + +var tableNameRegex = regexp.MustCompile(`^[a-zA-Z0-9_]+$`) + +const internalTablePrefix = "_devlake_" + +func ValidateTable(table string, allowedTables string) errors.Error { + if !tableNameRegex.MatchString(table) { + return errors.BadInput.New("table name invalid") + } + if strings.HasPrefix(table, internalTablePrefix) { + return errors.Forbidden.New("writing internal tables via push API is forbidden") + } + + allowlist := map[string]struct{}{} + for _, t := range strings.Split(allowedTables, ",") { + name := strings.TrimSpace(t) + if name == "" || strings.HasPrefix(name, internalTablePrefix) { + continue + } + allowlist[name] = struct{}{} + } + if len(allowlist) == 0 { + return errors.Forbidden.New("push API is disabled unless PUSH_API_ALLOWED_TABLES is configured") + } + if _, ok := allowlist[table]; !ok { + return errors.Forbidden.New("table name is not in the allowed list") + } + return nil +} diff --git a/backend/server/services/pushapiaccess/access_test.go b/backend/server/services/pushapiaccess/access_test.go new file mode 100644 index 00000000000..969a5436523 --- /dev/null +++ b/backend/server/services/pushapiaccess/access_test.go @@ -0,0 +1,70 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package pushapiaccess + +import "testing" + +func TestValidateTable(t *testing.T) { + cases := []struct { + name string + table string + allowedTables string + wantErr bool + }{ + { + name: "allows configured application table", + table: "commits", + allowedTables: "commits, issues", + }, + { + name: "rejects invalid table name", + table: "commits;drop", + allowedTables: "commits", + wantErr: true, + }, + { + name: "default denies when allowlist unset", + table: "commits", + wantErr: true, + }, + { + name: "rejects internal tables even when allowlisted", + table: "_devlake_pipelines", + allowedTables: "_devlake_pipelines,commits", + wantErr: true, + }, + { + name: "rejects tables missing from allowlist", + table: "pull_requests", + allowedTables: "commits,issues", + wantErr: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := ValidateTable(tc.table, tc.allowedTables) + if tc.wantErr && err == nil { + t.Fatal("expected an error but got nil") + } + if !tc.wantErr && err != nil { + t.Fatalf("expected no error, got %v", err) + } + }) + } +} diff --git a/backend/server/services/remote/bridge/context.go b/backend/server/services/remote/bridge/context.go index e66d1e66e25..08393dd465d 100644 --- a/backend/server/services/remote/bridge/context.go +++ b/backend/server/services/remote/bridge/context.go @@ -138,4 +138,11 @@ func (r remoteContextImpl) IncProgress(quantity int) { } } +func (r remoteContextImpl) GetProgress() int { + if r.parent != nil { + return r.parent.GetProgress() + } + return 0 +} + var _ RemoteContext = (*remoteContextImpl)(nil) diff --git a/backend/server/services/remote/models/conversion.go b/backend/server/services/remote/models/conversion.go index c3a3e0c6d24..7c44a1b9dc5 100644 --- a/backend/server/services/remote/models/conversion.go +++ b/backend/server/services/remote/models/conversion.go @@ -179,7 +179,7 @@ func getGoType(schema utils.JsonObject, required bool) (reflect.Type, errors.Err if required { return timeType, nil } else { - return reflect.PtrTo(timeType), nil + return reflect.PointerTo(timeType), nil } } else { return stringType, nil diff --git a/backend/test/Readme.md b/backend/test/Readme.md index 4987c9b0d6c..046bce5c99a 100644 --- a/backend/test/Readme.md +++ b/backend/test/Readme.md @@ -18,3 +18,26 @@ limitations under the License. * e2e/: Contains DevLake-server tests that interact with either fake plugins or no plugins at all. * integration/: Contains DevLake-server tests written against real data-sources which contain test data. + +### Running E2E tests locally + +Start a MySQL instance and create a test database that the `merico` user can access. The database URL must use +`127.0.0.1` instead of `localhost` because some Python dependencies do not resolve `localhost` correctly. + +```bash +export E2E_DB_URL='mysql://merico:merico@127.0.0.1:3306/lake_test?charset=utf8mb4&parseTime=True' + +cd backend +go run ./test/init.go +make e2e-test +``` + +The service E2E tests compile the `gitextractor` plugin, which links against `libgit2`. If `libgit2` is installed in +a non-standard location, expose it through `pkg-config` and add an rpath so the test binary can load the dynamic +library at runtime: + +```bash +export PKG_CONFIG_PATH=/path/to/libgit2/build +export CGO_LDFLAGS='-L/path/to/libgit2/build -Wl,-rpath,/path/to/libgit2/build' +make e2e-test +``` diff --git a/backend/test/e2e/services/server_startup_test.go b/backend/test/e2e/services/server_startup_test.go index f66c48aaeea..9432f80c263 100644 --- a/backend/test/e2e/services/server_startup_test.go +++ b/backend/test/e2e/services/server_startup_test.go @@ -39,6 +39,7 @@ import ( org "github.com/apache/incubator-devlake/plugins/org/impl" pagerduty "github.com/apache/incubator-devlake/plugins/pagerduty/impl" refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl" + rootly "github.com/apache/incubator-devlake/plugins/rootly/impl" slack "github.com/apache/incubator-devlake/plugins/slack/impl" sonarqube "github.com/apache/incubator-devlake/plugins/sonarqube/impl" starrocks "github.com/apache/incubator-devlake/plugins/starrocks/impl" @@ -78,6 +79,7 @@ func loadGoPlugins() []plugin.PluginMeta { org.Org{}, pagerduty.PagerDuty{}, refdiff.RefDiff{}, + rootly.Rootly{}, slack.Slack{}, sonarqube.Sonarqube{}, starrocks.StarRocks{}, diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go index 0af014b0513..8e39d6f5b01 100644 --- a/backend/test/helper/client.go +++ b/backend/test/helper/client.go @@ -137,6 +137,10 @@ func ConnectLocalServer(t *testing.T, clientConfig *LocalClientConfig) *DevlakeC fmt.Printf("Using test temp directory: %s\n", throwawayDir) logger := logruslog.Global.Nested("test") cfg := config.GetConfig() + // E2E helpers issue direct API requests without session or API-key auth. + // Keep local test servers aligned with that contract. + t.Setenv("AUTH_ENABLED", "false") + cfg.Set("AUTH_ENABLED", false) cfg.Set("DB_URL", clientConfig.DbURL) db, err := runner.NewGormDb(cfg, logger) require.NoError(t, err) @@ -176,6 +180,7 @@ func ConnectLocalServer(t *testing.T, clientConfig *LocalClientConfig) *DevlakeC api.CreateAndRunApiServer() }) }() + // NOTE: /proceed-db-migration now requires authentication. If AUTH_ENABLED=true, this test must provide credentials. req, err2 := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/proceed-db-migration", addr), nil) require.NoError(t, err2) d.forceSendHttpRequest(100, req, func(err errors.Error) bool { @@ -345,7 +350,7 @@ func runWithTimeout(timeout time.Duration, f func() (bool, errors.Error)) errors select { case <-timer: if !resp.completed { - return errors.Default.New(fmt.Sprintf("timed out calling function after %d miliseconds", timeout.Milliseconds())) + return errors.Default.New(fmt.Sprintf("timed out calling function after %d milliseconds", timeout.Milliseconds())) } return nil case resp = <-resChan: diff --git a/config-ui/Dockerfile b/config-ui/Dockerfile index 7fadb956bda..10b9d573bac 100644 --- a/config-ui/Dockerfile +++ b/config-ui/Dockerfile @@ -23,7 +23,7 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM node:18-bookworm-slim as builder +FROM --platform=$BUILDPLATFORM node:22-bookworm-slim AS builder WORKDIR /home/node/code COPY . . @@ -31,7 +31,7 @@ COPY . . RUN yarn install RUN yarn build -FROM nginxinc/nginx-unprivileged:1.29 +FROM nginxinc/nginx-unprivileged:1.30.3 USER 0 #ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait #RUN chmod +x /wait @@ -47,4 +47,4 @@ RUN chmod +x /usr/bin/nginx.sh RUN chgrp -R 0 /etc/nginx /var/log/nginx /usr/share/nginx /run && \ chmod -R g=u /etc/nginx /var/log/nginx /usr/share/nginx /run USER 101 -CMD /usr/bin/nginx.sh +CMD ["/usr/bin/nginx.sh"] diff --git a/config-ui/env.example b/config-ui/env.example index 078e902a541..1059e19df97 100644 --- a/config-ui/env.example +++ b/config-ui/env.example @@ -30,3 +30,4 @@ DEVLAKE_DASHBOARD_URL_GITHUB=/grafana/d/KXWvOFQnz/github?orgId=1&var-repo_id=All DEVLAKE_DASHBOARD_URL_GITLAB=/grafana/d/msSjEq97z/gitlab?orgId=1&var-repo_id=All&var-interval=WEEKDAY DEVLAKE_DASHBOARD_URL_BITBUCKET=/grafana/d/4LzQHZa4k/bitbucket?orgId=1&var-repo_id=All&var-interval=WEEKDAY DEVLAKE_DASHBOARD_URL_AZUREDEVOPS=/grafana/d/ba7e3a95-80ed-4067-a54b-2a82758eb3dd/azure-devops?orgId=1&var-repo_id=All&var-interval=WEEKDAY +DEVLAKE_DASHBOARD_URL_ASANA=/grafana/d/asana-dashboard?orgId=1 diff --git a/config-ui/package.json b/config-ui/package.json index c10c1c9d9ec..cd00e507e87 100644 --- a/config-ui/package.json +++ b/config-ui/package.json @@ -4,7 +4,7 @@ "version": "0.0.0", "packageManager": "yarn@3.4.1", "scripts": { - "start": "vite", + "start": "vite --host --strictPort", "build": "vite build", "preview": "vite preview", "lint": "tsc && eslint . --fix", diff --git a/config-ui/src/api/auth/index.ts b/config-ui/src/api/auth/index.ts new file mode 100644 index 00000000000..96b3b046a65 --- /dev/null +++ b/config-ui/src/api/auth/index.ts @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { request } from '@/utils'; + +export type Provider = { + name: string; + displayName: string; + loginUrl: string; +}; + +export type Methods = { + providers?: Provider[]; + apiKey?: { + enabled: boolean; + }; +}; + +export type UserInfo = { + authenticated: boolean; + name: string; + email: string; +}; + +export const methods = (): Promise => request('/auth/methods'); + +export const userinfo = (): Promise => request('/auth/userinfo'); + +export const logout = (): Promise<{ ok: boolean; logoutUrl?: string }> => request('/auth/logout', { method: 'POST' }); diff --git a/config-ui/src/api/connection/index.ts b/config-ui/src/api/connection/index.ts index 5a1ff5f96aa..2728b103ecc 100644 --- a/config-ui/src/api/connection/index.ts +++ b/config-ui/src/api/connection/index.ts @@ -50,9 +50,13 @@ export const test = ( | 'appId' | 'secretKey' | 'proxy' + | 'rateLimitPerHour' | 'dbUrl' | 'companyId' + | 'adminApiKey' | 'organization' + | 'organizationId' + | 'customHeaders' > >, ): Promise => @@ -70,7 +74,11 @@ export const testOld = ( | 'appId' | 'secretKey' | 'proxy' + | 'rateLimitPerHour' | 'dbUrl' + | 'adminApiKey' | 'organization' + | 'organizationId' + | 'customHeaders' >, ): Promise => request(`/plugins/${plugin}/test`, { method: 'post', data: payload }); diff --git a/config-ui/src/api/index.ts b/config-ui/src/api/index.ts index 2286d1afbae..b997d827893 100644 --- a/config-ui/src/api/index.ts +++ b/config-ui/src/api/index.ts @@ -19,6 +19,7 @@ import { request } from '@/utils'; import * as apiKey from './api-key'; +import * as auth from './auth'; import * as blueprint from './blueprint'; import * as connection from './connection'; import * as metricSettings from './metric-settings'; @@ -36,6 +37,7 @@ const version = (signal?: AbortSignal): Promise<{ version: string }> => request( export const API = { apiKey, + auth, blueprint, connection, metricSettings, diff --git a/config-ui/src/app/routrer.tsx b/config-ui/src/app/router.tsx similarity index 98% rename from config-ui/src/app/routrer.tsx rename to config-ui/src/app/router.tsx index 9bf0e401d83..fcb5d7b4714 100644 --- a/config-ui/src/app/routrer.tsx +++ b/config-ui/src/app/router.tsx @@ -24,6 +24,7 @@ import { Error, Layout, layoutLoader, + Login, Connections, Connection, ProjectHomePage, @@ -54,6 +55,10 @@ export const router = createBrowserRouter([ path: `${PATH_PREFIX}/db-migrate`, element: , }, + { + path: `${PATH_PREFIX}/login`, + element: , + }, { path: `${PATH_PREFIX}/onboard`, element: , diff --git a/config-ui/src/app/store.ts b/config-ui/src/app/store.ts index 8d2ce0c3429..6c3cfa3a55d 100644 --- a/config-ui/src/app/store.ts +++ b/config-ui/src/app/store.ts @@ -18,11 +18,12 @@ import { configureStore, ThunkAction, Action } from '@reduxjs/toolkit'; -import { connectionsSlice } from '@/features'; +import { connectionsSlice, themeSlice } from '@/features'; export const store = configureStore({ reducer: { connections: connectionsSlice.reducer, + theme: themeSlice.reducer, }, }); diff --git a/config-ui/src/components/block/styled.ts b/config-ui/src/components/block/styled.ts index 291e5b3e977..b66fa2ca8a9 100644 --- a/config-ui/src/components/block/styled.ts +++ b/config-ui/src/components/block/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const subLabel = styled.p` diff --git a/config-ui/src/components/inspector/styled.ts b/config-ui/src/components/inspector/styled.ts index c4bc25cb003..5526875a90c 100644 --- a/config-ui/src/components/inspector/styled.ts +++ b/config-ui/src/components/inspector/styled.ts @@ -20,7 +20,7 @@ import styled from 'styled-components'; export const Wrapper = styled.div` padding: 16px 24px; - background-color: #f3f3f3; + background-color: ${({ theme }) => theme.colors.bgElevated}; .title { display: flex; @@ -34,7 +34,7 @@ export const Wrapper = styled.div` span { font-size: 10px; - color: #aaaaaa; + color: ${({ theme }) => theme.colors.textFaint}; } } @@ -46,7 +46,7 @@ export const Wrapper = styled.div` .content { padding: 10px; max-height: 600; - background-color: #ffff; + background-color: ${({ theme }) => theme.colors.bgContainer}; border-radius: 4px; box-shadow: 1px 1px 3px 0px rgb(0 0 0 / 20%) inset; overflow-y: auto; diff --git a/config-ui/src/components/loading/styled.ts b/config-ui/src/components/loading/styled.ts index b5c5ce82f29..2badd833024 100644 --- a/config-ui/src/components/loading/styled.ts +++ b/config-ui/src/components/loading/styled.ts @@ -37,7 +37,7 @@ export const Wrapper = styled.div` export const Spin = styled.div<{ size: number }>` width: ${({ size }) => size}px; height: ${({ size }) => size}px; - border: 2px solid #7497f7; + border: 2px solid ${({ theme }) => theme.colors.primary}; border-radius: 50%; border-right-color: transparent; box-sizing: border-box; diff --git a/config-ui/src/components/message/index.tsx b/config-ui/src/components/message/index.tsx index 6b9f2fc2fed..ef7d633e158 100644 --- a/config-ui/src/components/message/index.tsx +++ b/config-ui/src/components/message/index.tsx @@ -35,7 +35,7 @@ export const Message = ({ style, size = 20, content }: Props) => { return ( - + {content} diff --git a/config-ui/src/components/page-header/styled.ts b/config-ui/src/components/page-header/styled.ts index 297e2dedfa5..2b031208273 100644 --- a/config-ui/src/components/page-header/styled.ts +++ b/config-ui/src/components/page-header/styled.ts @@ -47,7 +47,7 @@ export const Breadcrumb = styled.li` a { display: flex; align-items: center; - color: #292b3f; + color: ${({ theme }) => theme.colors.text}; } `; export const Extra = styled.div``; diff --git a/config-ui/src/components/tip-layout/styled.ts b/config-ui/src/components/tip-layout/styled.ts index 14dd42a74f4..b3f04dfb861 100644 --- a/config-ui/src/components/tip-layout/styled.ts +++ b/config-ui/src/components/tip-layout/styled.ts @@ -24,7 +24,7 @@ export const Wrapper = styled.div` align-items: center; padding-top: 100px; height: 100vh; - background-color: #f9f9fa; + background-color: ${({ theme }) => theme.colors.bgLayout}; box-sizing: border-box; `; @@ -42,7 +42,7 @@ export const Inner = styled.div` margin: 16px 0; &.warning { - color: #faad14; + color: ${({ theme }) => theme.colors.warningAlt}; } } `; diff --git a/config-ui/src/config/entities.ts b/config-ui/src/config/entities.ts index c0ca9ccbb3c..edc2c125e51 100644 --- a/config-ui/src/config/entities.ts +++ b/config-ui/src/config/entities.ts @@ -22,6 +22,7 @@ export const EntitiesLabel: Record = { CODEREVIEW: 'Code Review', CICD: 'CI/CD', CROSS: 'Cross Domain', + CLAUDE_CODE: 'Claude Code', CODEQUALITY: 'Code Quality Domain', }; diff --git a/config-ui/src/features/connections/utils.ts b/config-ui/src/features/connections/utils.ts index 792213817ae..88cf1c692d7 100644 --- a/config-ui/src/features/connections/utils.ts +++ b/config-ui/src/features/connections/utils.ts @@ -23,23 +23,11 @@ import { IConnectionAPI, IConnection, IConnectionStatus, IWebhookAPI, IWebhook } export const transformConnection = (plugin: string, connection: IConnectionAPI): IConnection => { const config = getPluginConfig(plugin); return { + ...connection, unique: `${plugin}-${connection.id}`, plugin, pluginName: config.name, - id: connection.id, - name: connection.name, status: IConnectionStatus.IDLE, - endpoint: connection.endpoint, - authMethod: connection.authMethod, - token: connection.token, - username: connection.username, - password: connection.password, - appId: connection.appId, - secretKey: connection.secretKey, - dbUrl: connection.dbUrl, - proxy: connection.proxy, - rateLimitPerHour: connection.rateLimitPerHour, - organization: connection.organization, }; }; diff --git a/config-ui/src/features/index.ts b/config-ui/src/features/index.ts index 4e6e8a4dece..4a64f6e0419 100644 --- a/config-ui/src/features/index.ts +++ b/config-ui/src/features/index.ts @@ -17,3 +17,4 @@ */ export * from './connections'; +export * from './theme'; diff --git a/config-ui/src/features/theme/index.ts b/config-ui/src/features/theme/index.ts new file mode 100644 index 00000000000..513ab48a7f8 --- /dev/null +++ b/config-ui/src/features/theme/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './slice'; diff --git a/config-ui/src/features/theme/slice.ts b/config-ui/src/features/theme/slice.ts new file mode 100644 index 00000000000..cd39897324e --- /dev/null +++ b/config-ui/src/features/theme/slice.ts @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { createSlice, PayloadAction } from '@reduxjs/toolkit'; + +import { RootState } from '@/app/store'; +import type { ThemeMode, ResolvedTheme } from '@/theme/tokens'; + +export const THEME_STORAGE_KEY = 'devlake.theme'; + +const NEXT_MODE: Record = { + light: 'dark', + dark: 'system', + system: 'light', +}; + +const isThemeMode = (value: unknown): value is ThemeMode => value === 'light' || value === 'dark' || value === 'system'; + +const loadInitialMode = (): ThemeMode => { + if (typeof window === 'undefined') return 'system'; + try { + const stored = window.localStorage.getItem(THEME_STORAGE_KEY); + if (isThemeMode(stored)) return stored; + } catch { + // localStorage may be unavailable (e.g., SSR / privacy mode) + } + return 'system'; +}; + +const persistMode = (mode: ThemeMode): void => { + if (typeof window === 'undefined') return; + try { + window.localStorage.setItem(THEME_STORAGE_KEY, mode); + } catch { + // localStorage may be unavailable (e.g., SSR / privacy mode) + } +}; + +interface ThemeState { + mode: ThemeMode; +} + +const initialState: ThemeState = { + mode: loadInitialMode(), +}; + +export const themeSlice = createSlice({ + name: 'theme', + initialState, + reducers: { + setMode(state, action: PayloadAction) { + state.mode = action.payload; + persistMode(action.payload); + }, + cycleMode(state) { + state.mode = NEXT_MODE[state.mode]; + persistMode(state.mode); + }, + }, +}); + +export const { setMode, cycleMode } = themeSlice.actions; + +export default themeSlice.reducer; + +export const selectThemeMode = (state: RootState): ThemeMode => state.theme.mode; + +export const resolveThemeMode = (mode: ThemeMode): ResolvedTheme => { + if (mode !== 'system') return mode; + if (typeof window === 'undefined' || typeof window.matchMedia !== 'function') return 'light'; + return window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'; +}; diff --git a/config-ui/src/hooks/use-refresh-data.ts b/config-ui/src/hooks/use-refresh-data.ts index 5e1fa7a0f84..be9776fec3d 100644 --- a/config-ui/src/hooks/use-refresh-data.ts +++ b/config-ui/src/hooks/use-refresh-data.ts @@ -26,7 +26,7 @@ export const useRefreshData = (request: (signal: AbortSignal) => Promise, state: 'ready' | 'pending' | 'error'; deps?: React.DependencyList; data?: T; - error?: unknown; + error?: unknown; abortController?: AbortController; timer?: number; }>({ @@ -38,7 +38,7 @@ export const useRefreshData = (request: (signal: AbortSignal) => Promise, data: ref.current.data, ready: ref.current.state === 'ready', pending: ref.current.state === 'pending', - error: ref.current.error, + error: ref.current.error, }; } diff --git a/config-ui/src/index.css b/config-ui/src/index.css index c322bc785f7..7e523eb049b 100644 --- a/config-ui/src/index.css +++ b/config-ui/src/index.css @@ -16,8 +16,16 @@ * */ +/* Initial-paint fallbacks; ThemeProvider overrides these from theme tokens. */ +:root { + --devlake-color-text: #292b3f; + --devlake-color-text-muted: #94959f; + --devlake-color-bg: #ffffff; +} + body { - color: #292b3f; + color: var(--devlake-color-text); + background-color: var(--devlake-color-bg); margin: 0; } @@ -64,7 +72,7 @@ ul { p { margin: 8px 0; font-size: 12px; - color: #94959f; + color: var(--devlake-color-text-muted); } #root { diff --git a/config-ui/src/main.tsx b/config-ui/src/main.tsx index d3b35c690ac..eeff8947e49 100644 --- a/config-ui/src/main.tsx +++ b/config-ui/src/main.tsx @@ -19,25 +19,19 @@ import ReactDOM from 'react-dom'; import { Provider } from 'react-redux'; import { RouterProvider } from 'react-router-dom'; -import { ConfigProvider } from 'antd'; import { PageLoading } from '@/components'; +import { ThemeProvider } from '@/theme'; import { store } from './app/store'; -import { router } from './app/routrer'; +import { router } from './app/router'; import './index.css'; ReactDOM.render( - - + + } /> - - , + + , document.getElementById('root'), ); diff --git a/config-ui/src/plugins/components/connection-form/fields/styled.ts b/config-ui/src/plugins/components/connection-form/fields/styled.ts index 08b4e92677f..1c69cdbe320 100644 --- a/config-ui/src/plugins/components/connection-form/fields/styled.ts +++ b/config-ui/src/plugins/components/connection-form/fields/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` diff --git a/config-ui/src/plugins/components/connection-form/index.tsx b/config-ui/src/plugins/components/connection-form/index.tsx index 818f70d09e5..e846c278c55 100644 --- a/config-ui/src/plugins/components/connection-form/index.tsx +++ b/config-ui/src/plugins/components/connection-form/index.tsx @@ -26,10 +26,70 @@ import { ExternalLink } from '@/components'; import { addConnection, updateConnection } from '@/features'; import { selectConnection } from '@/features/connections'; import { getPluginConfig } from '@/plugins'; +import { ICustomHeader } from '@/types'; import { operator } from '@/utils'; import { Form } from './fields'; +const sanitizeCustomHeaders = (headers?: Array<{ key?: string; value?: string }>): ICustomHeader[] | undefined => { + if (!headers) { + return headers; + } + + return headers + .filter((header) => header.key?.trim() || header.value?.trim()) + .map((header) => ({ + key: header.key?.trim() ?? '', + value: header.value ?? '', + })); +}; + +const buildUpdateTestPayload = (connection: any, values: any, customHeaders: ICustomHeader[] | undefined) => ({ + endpoint: isEqual(connection?.endpoint, values.endpoint) ? undefined : values.endpoint, + authMethod: isEqual(connection?.authMethod, values.authMethod) ? undefined : values.authMethod, + username: isEqual(connection?.username, values.username) ? undefined : values.username, + password: isEqual(connection?.password, values.password) ? undefined : values.password, + token: isEqual(connection?.token, values.token) ? undefined : values.token, + appId: isEqual(connection?.appId, values.appId) ? undefined : values.appId, + secretKey: isEqual(connection?.secretKey, values.secretKey) ? undefined : values.secretKey, + proxy: isEqual(connection?.proxy, values.proxy) ? undefined : values.proxy, + rateLimitPerHour: isEqual(connection?.rateLimitPerHour, values.rateLimitPerHour) + ? undefined + : values.rateLimitPerHour, + dbUrl: isEqual(connection?.dbUrl, values.dbUrl) ? undefined : values.dbUrl, + companyId: isEqual(connection?.companyId, values.companyId) ? undefined : values.companyId, + organization: isEqual(connection?.organization, values.organization) ? undefined : values.organization, + enterprise: isEqual(connection?.enterprise, values.enterprise) ? undefined : values.enterprise, + customHeaders: isEqual(connection?.customHeaders, customHeaders) ? undefined : customHeaders, +}); + +const buildCreateTestPayload = (initialValues: any, values: any, customHeaders: ICustomHeader[] | undefined) => + pick({ ...initialValues, ...values, customHeaders }, [ + 'name', + 'endpoint', + 'token', + 'username', + 'password', + 'proxy', + 'authMethod', + 'appId', + 'secretKey', + 'accessKeyId', + 'secretAccessKey', + 'region', + 'bucket', + 'identityStoreId', + 'identityStoreRegion', + 'rateLimitPerHour', + 'tenantId', + 'tenantType', + 'dbUrl', + 'companyId', + 'organization', + 'enterprise', + 'customHeaders', + ]); + interface Props { plugin: string; connectionId?: ID; @@ -51,56 +111,30 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { const { name, - connection: { docLink, fields, initialValues }, - } = getPluginConfig(plugin); + connection: { docLink, fields, initialValues = {} }, + } = getPluginConfig(plugin) ?? {}; const disabled = useMemo(() => { - return Object.values(errors).some((value) => value); + return Object.values(errors).some(Boolean); }, [errors]); + const sanitizedCustomHeaders = useMemo(() => sanitizeCustomHeaders(values.customHeaders), [values.customHeaders]); + + if (!plugin || !name) { + return null; + } + const handleTest = async () => { + const isUpdate = type === 'update' && !!connectionId; await operator( () => - type === 'update' && connectionId - ? API.connection.test(plugin, connectionId, { - endpoint: isEqual(connection?.endpoint, values.endpoint) ? undefined : values.endpoint, - authMethod: isEqual(connection?.authMethod, values.authMethod) ? undefined : values.authMethod, - username: isEqual(connection?.username, values.username) ? undefined : values.username, - password: isEqual(connection?.password, values.password) ? undefined : values.password, - token: isEqual(connection?.token, values.token) ? undefined : values.token, - appId: isEqual(connection?.appId, values.appId) ? undefined : values.appId, - secretKey: isEqual(connection?.secretKey, values.secretKey) ? undefined : values.secretKey, - proxy: isEqual(connection?.proxy, values.proxy) ? undefined : values.proxy, - dbUrl: isEqual(connection?.dbUrl, values.dbUrl) ? undefined : values.dbUrl, - companyId: isEqual(connection?.companyId, values.companyId) ? undefined : values.companyId, - organization: isEqual(connection?.organization, values.organization) ? undefined : values.organization, - }) - : API.connection.testOld( + isUpdate + ? API.connection.test( plugin, - pick({ ...initialValues, ...values }, [ - 'name', - 'endpoint', - 'token', - 'username', - 'password', - 'proxy', - 'authMethod', - 'appId', - 'secretKey', - 'accessKeyId', - 'secretAccessKey', - 'region', - 'bucket', - 'identityStoreId', - 'identityStoreRegion', - 'rateLimitPerHour', - 'tenantId', - 'tenantType', - 'dbUrl', - 'companyId', - 'organization', - ]), - ), + connectionId, + buildUpdateTestPayload(connection, values, sanitizedCustomHeaders), + ) + : API.connection.testOld(plugin, buildCreateTestPayload(initialValues, values, sanitizedCustomHeaders)), { setOperating, formatMessage: () => 'Test Connection Successfully.', @@ -109,14 +143,17 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { }; const handleSave = async () => { + const isCreate = !connectionId; const [success, res] = await operator( () => - !connectionId - ? dispatch(addConnection({ plugin, ...values })).unwrap() - : dispatch(updateConnection({ plugin, connectionId, ...values })).unwrap(), + isCreate + ? dispatch(addConnection({ plugin, ...values, customHeaders: sanitizedCustomHeaders })).unwrap() + : dispatch( + updateConnection({ plugin, connectionId, ...values, customHeaders: sanitizedCustomHeaders }), + ).unwrap(), { setOperating, - formatMessage: () => (!connectionId ? 'Create a New Connection Successful.' : 'Update Connection Successful.'), + formatMessage: () => (isCreate ? 'Create a New Connection Successful.' : 'Update Connection Successful.'), }, ); @@ -140,7 +177,7 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { type={type} name={name} fields={fields} - initialValues={{ ...initialValues, ...(connection ?? {}) }} + initialValues={connection ? { ...initialValues, ...connection } : initialValues} values={values} errors={errors} setValues={setValues} diff --git a/config-ui/src/plugins/components/connection-list/index.tsx b/config-ui/src/plugins/components/connection-list/index.tsx index e0985b1d14d..2b253f25eb1 100644 --- a/config-ui/src/plugins/components/connection-list/index.tsx +++ b/config-ui/src/plugins/components/connection-list/index.tsx @@ -113,7 +113,7 @@ export const ConnectionList = ({ plugin, onCreate }: Props) => { dataSource={connections} pagination={false} /> - onChange(selectedScope.filter((it) => it.id !== sc.id))} > - {sc.fullName ?? sc.name} + {getPluginScopeName(plugin, sc) || sc.fullName || sc.name || sc.id} )) ) : ( @@ -218,7 +219,7 @@ export const SearchLocal = ({ mode, plugin, connectionId, config, disabledScope, {status === 'loaded' && ( - + {miller.items.length} scopes found )} diff --git a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx index 4679b856409..5f61d74106b 100644 --- a/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx +++ b/config-ui/src/plugins/components/data-scope-remote/search-remote.tsx @@ -27,6 +27,7 @@ import { uniqBy } from 'lodash'; import API from '@/api'; import { Loading, Block } from '@/components'; import { IPluginConfig } from '@/types'; +import { getPluginScopeName } from '@/plugins'; import * as T from './types'; import * as S from './styled'; @@ -53,6 +54,8 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope nextTokenMap: {}, }); + const PAGE_SIZE = 50; + const [search, setSearch] = useState<{ loading: boolean; items: McsItem[]; @@ -60,6 +63,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope query: string; page: number; total: number; + hasMore: boolean; }>({ loading: true, items: [], @@ -67,6 +71,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope query: '', page: 1, total: 0, + hasMore: false, }); const searchDebounce = useDebounce(search.query, { wait: 500 }); @@ -93,7 +98,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope newItems = (res.children ?? []).map((it) => ({ ...it, - title: it.name, + title: getPluginScopeName(plugin, it) || it.name, })); nextPageToken = res.nextPageToken; @@ -128,24 +133,35 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope const searchItems = async () => { if (!searchDebounce) return; - const res = await API.scope.searchRemote(plugin, connectionId, { - search: searchDebounce, - page: search.page, - pageSize: 20, - }); - - const newItems = (res.children ?? []).map((it) => ({ - ...it, - title: it.fullName ?? it.name, - })); - - setSearch((s) => ({ - ...s, - loading: false, - items: [...allItems, ...newItems], - currentItems: newItems, - total: res.count, - })); + try { + const res = await API.scope.searchRemote(plugin, connectionId, { + search: searchDebounce, + page: search.page, + pageSize: PAGE_SIZE, + }); + + const newItems = (res.children ?? []).map((it) => ({ + ...it, + title: getPluginScopeName(plugin, it) || it.fullName || it.name, + })); + + const total = res.count ?? 0; + // If the backend returns a real total, use it; otherwise fall back to + // the heuristic: a full page means there are likely more results. + const hasMore = total > 0 ? search.page * PAGE_SIZE < total : newItems.length >= PAGE_SIZE; + + setSearch((s) => ({ + ...s, + loading: false, + items: [...allItems, ...newItems], + // Accumulate results across pages so previous pages remain visible + currentItems: s.page === 1 ? newItems : [...s.currentItems, ...newItems], + total, + hasMore, + })); + } catch { + setSearch((s) => ({ ...s, loading: false, hasMore: false })); + } }; useEffect(() => { @@ -164,7 +180,7 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope closable onClose={() => onChange(selectedScope.filter((it) => it.id !== sc.id))} > - {sc.fullName} + {getPluginScopeName(plugin, sc) || sc.fullName || sc.name || sc.id} )) ) : ( @@ -177,7 +193,9 @@ export const SearchRemote = ({ mode, plugin, connectionId, config, disabledScope prefix={} placeholder={config.searchPlaceholder ?? 'Search'} value={search.query} - onChange={(e) => setSearch({ ...search, query: e.target.value, loading: true, currentItems: [] })} + onChange={(e) => + setSearch({ ...search, query: e.target.value, loading: true, currentItems: [], page: 1, hasMore: false }) + } /> {!searchDebounce ? ( false} - getHasMore={() => search.loading} - onScroll={() => setSearch({ ...search, page: search.page + 1 })} + getHasMore={() => search.hasMore} + onScroll={() => { + if (!search.loading && search.hasMore) { + setSearch((s) => ({ ...s, loading: true, page: s.page + 1 })); + } + }} renderLoading={() => } disabledIds={(disabledScope ?? []).map((it) => it.id)} selectedIds={selectedScope.map((it) => it.id)} diff --git a/config-ui/src/plugins/components/data-scope-remote/styled.ts b/config-ui/src/plugins/components/data-scope-remote/styled.ts index 27ecd0fe893..dc880b18c07 100644 --- a/config-ui/src/plugins/components/data-scope-remote/styled.ts +++ b/config-ui/src/plugins/components/data-scope-remote/styled.ts @@ -31,6 +31,6 @@ export const JobLoad = styled.div` & > span.count { margin: 0 8px; - color: #7497f7; + color: ${({ theme }) => theme.colors.primary}; } `; diff --git a/config-ui/src/plugins/components/data-scope-select/index.tsx b/config-ui/src/plugins/components/data-scope-select/index.tsx index 79bd8b082f9..860cce6cd06 100644 --- a/config-ui/src/plugins/components/data-scope-select/index.tsx +++ b/config-ui/src/plugins/components/data-scope-select/index.tsx @@ -27,7 +27,7 @@ import API from '@/api'; import { PATHS } from '@/config'; import { Loading, Block, ExternalLink, Message } from '@/components'; import { useRefreshData } from '@/hooks'; -import { getPluginScopeId } from '@/plugins'; +import { getPluginScopeId, getPluginScopeName } from '@/plugins'; interface Props { plugin: string; @@ -70,7 +70,7 @@ export const DataScopeSelect = ({ ...res.scopes.map((sc) => ({ parentId: null, id: getPluginScopeId(plugin, sc.scope), - title: sc.scope.fullName ?? sc.scope.name, + title: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, data: sc.scope, })), ]); @@ -93,7 +93,7 @@ export const DataScopeSelect = ({ const searchOptions = useMemo( () => data?.scopes.map((sc) => ({ - label: sc.scope.fullName ?? sc.scope.name, + label: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, value: getPluginScopeId(plugin, sc.scope), })) ?? [], [data], diff --git a/config-ui/src/plugins/components/scope-config-form/index.tsx b/config-ui/src/plugins/components/scope-config-form/index.tsx index 0f45a5708ff..46fbf129a59 100644 --- a/config-ui/src/plugins/components/scope-config-form/index.tsx +++ b/config-ui/src/plugins/components/scope-config-form/index.tsx @@ -35,6 +35,8 @@ import { TapdTransformation } from '@/plugins/register/tapd'; import { BambooTransformation } from '@/plugins/register/bamboo'; import { CircleCITransformation } from '@/plugins/register/circleci'; import { ArgoCDTransformation } from '@/plugins/register/argocd'; +import { GhCopilotTransformation } from '@/plugins/register/gh-copilot'; +import { AsanaTransformation } from '@/plugins/register/asana'; import { DOC_URL } from '@/release'; import { operator } from '@/utils'; @@ -70,6 +72,11 @@ export const ScopeConfigForm = ({ const config = useMemo(() => getPluginConfig(plugin), []); + const pluginDoc = DOC_URL.PLUGIN[config.plugin.toUpperCase() as keyof typeof DOC_URL.PLUGIN]; + + const transformationDoc = + typeof pluginDoc === 'object' && pluginDoc && 'TRANSFORMATION' in pluginDoc ? pluginDoc.TRANSFORMATION : undefined; + useEffect(() => { setTransformation(config.scopeConfig?.transformation ?? {}); }, [config.scopeConfig?.transformation]); @@ -87,7 +94,7 @@ export const ScopeConfigForm = ({ setName(forceCreate ? `${res.name}-copy` : res.name); setEntities(res.entities ?? []); setTransformation(omit(res, ['id', 'connectionId', 'name', 'entities', 'createdAt', 'updatedAt'])); - } catch { } + } catch {} })(); }, [scopeConfigId]); @@ -118,18 +125,16 @@ export const ScopeConfigForm = ({ return ( - - To learn about how {config.name} transformation is used in DevLake, - {/* @ts-ignore */} - - check out this doc - - . - - } - /> + {transformationDoc && ( + + To learn about how {config.name} transformation is used in DevLake, + check out this doc. + + } + /> + )} {step === 1 && ( <> @@ -259,6 +264,14 @@ export const ScopeConfigForm = ({ /> )} + {plugin === 'gh-copilot' && ( + + )} + {plugin === 'gitlab' && ( )} + {plugin === 'asana' && ( + + )} + {plugin === 'tapd' && scopeId && ( - * + *
@@ -136,9 +136,9 @@ const renderCollapseItems = ({
- Note: ArgoCD limits deployment history to the last 10 sync operations by default (controlled - by revisionHistoryLimit). Consider increasing this value in your ArgoCD application settings for - better historical metrics. + Note: ArgoCD limits deployment history to the last 10 sync operations by default + (controlled by revisionHistoryLimit). Consider increasing this value in your ArgoCD application + settings for better historical metrics.
), diff --git a/config-ui/src/plugins/register/asana/assets/icon.svg b/config-ui/src/plugins/register/asana/assets/icon.svg new file mode 100644 index 00000000000..daf46f52ea4 --- /dev/null +++ b/config-ui/src/plugins/register/asana/assets/icon.svg @@ -0,0 +1,20 @@ + + + + + diff --git a/config-ui/src/plugins/register/asana/config.tsx b/config-ui/src/plugins/register/asana/config.tsx new file mode 100644 index 00000000000..af570c0e80e --- /dev/null +++ b/config-ui/src/plugins/register/asana/config.tsx @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { DOC_URL } from '@/release'; +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const AsanaConfig: IPluginConfig = { + plugin: 'asana', + name: 'Asana', + icon: ({ color }) => , + sort: 12, + connection: { + docLink: DOC_URL.PLUGIN.ASANA.BASIS, + initialValues: { + endpoint: 'https://app.asana.com/api/1.0/', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Asana API base URL.', + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for default.', + defaultValue: 150, + }, + ], + }, + dataScope: { + title: 'Projects', + millerColumn: { + columnCount: 4, + firstColumnTitle: 'Workspaces', + }, + searchPlaceholder: 'Search projects...', + }, + scopeConfig: { + entities: ['TICKET'], + transformation: { + issueTypeRequirement: '(feat|feature|story|requirement)', + issueTypeBug: '(bug|defect|broken)', + issueTypeIncident: '(incident|outage|failure)', + }, + }, +}; diff --git a/config-ui/src/plugins/register/asana/index.ts b/config-ui/src/plugins/register/asana/index.ts new file mode 100644 index 00000000000..5f16858cbe4 --- /dev/null +++ b/config-ui/src/plugins/register/asana/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/asana/transformation.tsx b/config-ui/src/plugins/register/asana/transformation.tsx new file mode 100644 index 00000000000..3277ee8e787 --- /dev/null +++ b/config-ui/src/plugins/register/asana/transformation.tsx @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { CaretRightOutlined } from '@ant-design/icons'; +import { theme, Collapse, Tag, Form, Input } from 'antd'; + +import { ExternalLink } from '@/components'; +import { DOC_URL } from '@/release'; + +interface Props { + entities: string[]; + connectionId: ID; + transformation: any; + setTransformation: React.Dispatch>; +} + +export const AsanaTransformation = ({ entities, transformation, setTransformation }: Props) => { + const { token } = theme.useToken(); + + const panelStyle: React.CSSProperties = { + marginBottom: 24, + background: token.colorFillAlter, + borderRadius: token.borderRadiusLG, + border: 'none', + }; + + return ( + } + style={{ background: token.colorBgContainer }} + size="large" + items={[ + { + key: 'TICKET', + label: 'Issue Tracking', + style: panelStyle, + children: ( + <> +

+ Tell DevLake what your Asana tags mean to view metrics such as{' '} + Bug Age,{' '} + DORA - Median Time to Restore Service, etc. +

+

+ DevLake defines three standard types of issues: REQUIREMENT, BUG and INCIDENT. Classify your Asana tasks + using tags that match the RegEx patterns below. +

+ + + setTransformation({ + ...transformation, + issueTypeRequirement: e.target.value, + }) + } + /> + + + + setTransformation({ + ...transformation, + issueTypeBug: e.target.value, + }) + } + /> + + + Incident + + DORA + + + } + > + + setTransformation({ + ...transformation, + issueTypeIncident: e.target.value, + }) + } + /> + + + ), + }, + ].filter((it) => entities.includes(it.key))} + /> + ); +}; diff --git a/config-ui/src/plugins/register/azure/config.tsx b/config-ui/src/plugins/register/azure/config.tsx index 4d726f1ec26..a3f574d615a 100644 --- a/config-ui/src/plugins/register/azure/config.tsx +++ b/config-ui/src/plugins/register/azure/config.tsx @@ -30,6 +30,7 @@ export const AzureConfig: IPluginConfig = { sort: 2, connection: { docLink: DOC_URL.PLUGIN.AZUREDEVOPS.BASIS, + initialValues: {}, fields: [ 'name', () => , @@ -82,6 +83,7 @@ export const AzureGoConfig: IPluginConfig = { isBeta: true, connection: { docLink: DOC_URL.PLUGIN.AZUREDEVOPS.BASIS, + initialValues: {}, fields: [ 'name', () => , diff --git a/config-ui/src/plugins/register/azure/connection-fields/styled.ts b/config-ui/src/plugins/register/azure/connection-fields/styled.ts index 11e47a3d0a1..acdd9a72879 100644 --- a/config-ui/src/plugins/register/azure/connection-fields/styled.ts +++ b/config-ui/src/plugins/register/azure/connection-fields/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` diff --git a/config-ui/src/plugins/register/azure/transformation.tsx b/config-ui/src/plugins/register/azure/transformation.tsx index df57ceb04fb..1a8ea080740 100644 --- a/config-ui/src/plugins/register/azure/transformation.tsx +++ b/config-ui/src/plugins/register/azure/transformation.tsx @@ -100,7 +100,7 @@ const renderCollapseItems = ({ }) } /> - * + *
diff --git a/config-ui/src/plugins/register/bamboo/config.tsx b/config-ui/src/plugins/register/bamboo/config.tsx index 9c0a614eb73..e30b0735e4e 100644 --- a/config-ui/src/plugins/register/bamboo/config.tsx +++ b/config-ui/src/plugins/register/bamboo/config.tsx @@ -28,6 +28,7 @@ export const BambooConfig: IPluginConfig = { sort: 3, connection: { docLink: DOC_URL.PLUGIN.BAMBOO.BASIS, + initialValues: {}, fields: [ 'name', 'endpoint', diff --git a/config-ui/src/plugins/register/bitbucket-server/config.tsx b/config-ui/src/plugins/register/bitbucket-server/config.tsx index f3b3c2bb097..6c979924915 100644 --- a/config-ui/src/plugins/register/bitbucket-server/config.tsx +++ b/config-ui/src/plugins/register/bitbucket-server/config.tsx @@ -29,6 +29,7 @@ export const BitbucketServerConfig: IPluginConfig = { isBeta: true, connection: { docLink: DOC_URL.PLUGIN.BITBUCKET_SERVER.BASIS, + initialValues: {}, fields: [ 'name', { diff --git a/config-ui/src/plugins/register/circleci/config.tsx b/config-ui/src/plugins/register/circleci/config.tsx index f6fcf06ccbe..64deb5f4ade 100644 --- a/config-ui/src/plugins/register/circleci/config.tsx +++ b/config-ui/src/plugins/register/circleci/config.tsx @@ -34,8 +34,10 @@ export const CircleCIConfig: IPluginConfig = { key: 'endpoint', multipleVersions: { cloud: 'https://circleci.com/api/', - server: '', + server: '(v4.x+)', }, + subLabel: + 'If you are using CircleCI Server, please enter the endpoint URL. E.g. https://circleci.your-company.com/api/', }, 'token', 'proxy', diff --git a/config-ui/src/plugins/register/circleci/transformation.tsx b/config-ui/src/plugins/register/circleci/transformation.tsx index 90a00377d15..0fce8f6f701 100644 --- a/config-ui/src/plugins/register/circleci/transformation.tsx +++ b/config-ui/src/plugins/register/circleci/transformation.tsx @@ -100,7 +100,7 @@ const renderCollapseItems = ({ }) } /> - * + *
diff --git a/config-ui/src/plugins/register/claude-code/assets/icon.svg b/config-ui/src/plugins/register/claude-code/assets/icon.svg new file mode 100644 index 00000000000..63abcb7c07d --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/assets/icon.svg @@ -0,0 +1,4 @@ + + + + diff --git a/config-ui/src/plugins/register/claude-code/config.tsx b/config-ui/src/plugins/register/claude-code/config.tsx new file mode 100644 index 00000000000..2c97200aee6 --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/config.tsx @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; +import { Token, Organization, CustomHeaders } from './connection-fields'; + +export const ClaudeCodeConfig: IPluginConfig = { + plugin: 'claude_code', + name: 'Claude Code', + icon: ({ color }) => , + sort: 6.6, + isBeta: true, + connection: { + docLink: 'https://github.com/apache/incubator-devlake/tree/main/backend/plugins/claude_code', + initialValues: { + endpoint: 'https://api.anthropic.com', + organization: '', + token: '', + customHeaders: [], + rateLimitPerHour: 1000, + }, + fields: [ + 'name', + 'endpoint', + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: + 'By default, DevLake uses 1,000 requests/hour for Claude Code usage collection. Adjust this value to throttle collection speed.', + defaultValue: 1000, + }, + ], + }, + dataScope: { + title: 'Organizations', + }, + scopeConfig: { + entities: ['CLAUDE_CODE'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/claude-code/connection-fields/custom-headers.tsx b/config-ui/src/plugins/register/claude-code/connection-fields/custom-headers.tsx new file mode 100644 index 00000000000..84b90f149f6 --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/connection-fields/custom-headers.tsx @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, type ChangeEvent } from 'react'; +import { Button, Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const CustomHeaders = ({ type, initialValues, values, setValues, setErrors }: Props) => { + const headers: Array<{ key: string; value: string }> = values.customHeaders ?? []; + const hasIncompleteHeaders = headers.some((header) => { + const hasKey = !!header.key?.trim(); + const hasValue = !!header.value?.trim(); + return hasKey !== hasValue; + }); + + useEffect(() => { + setValues({ + customHeaders: (initialValues.customHeaders ?? []).filter( + (header: { key: string; value: string }) => header.key?.trim() || header.value?.trim(), + ), + }); + }, [type, initialValues.customHeaders]); + + useEffect(() => { + setErrors({ + customHeaders: hasIncompleteHeaders ? 'Each custom header must include both a name and a value.' : '', + }); + }, [hasIncompleteHeaders]); + + const addHeader = () => { + setValues({ customHeaders: [...headers, { key: '', value: '' }] }); + }; + + const removeHeader = (index: number) => { + setValues({ customHeaders: headers.filter((_, i) => i !== index) }); + }; + + const updateHeader = (index: number, field: 'key' | 'value', newValue: string) => { + setValues({ + customHeaders: headers.map((h, i) => (i === index ? { ...h, [field]: newValue } : h)), + }); + }; + + return ( + + {headers.map((header, index) => ( +
+ ) => updateHeader(index, 'key', e.target.value)} + /> + ) => updateHeader(index, 'value', e.target.value)} + /> + +
+ ))} + {hasIncompleteHeaders && Each custom header must include both a name and a value.} + +
+ ); +}; diff --git a/config-ui/src/plugins/register/claude-code/connection-fields/index.ts b/config-ui/src/plugins/register/claude-code/connection-fields/index.ts new file mode 100644 index 00000000000..0382f03be5d --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/connection-fields/index.ts @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './custom-headers'; +export * from './organization'; +export * from './token'; diff --git a/config-ui/src/plugins/register/claude-code/connection-fields/organization.tsx b/config-ui/src/plugins/register/claude-code/connection-fields/organization.tsx new file mode 100644 index 00000000000..0d9276a482a --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/connection-fields/organization.tsx @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo, type ChangeEvent } from 'react'; +import { Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const Organization = ({ type, initialValues, values, setValues, setErrors }: Props) => { + useEffect(() => { + setValues({ organization: initialValues.organization ?? '' }); + }, [initialValues.organization]); + + const error = useMemo(() => { + return values.organization?.trim() ? '' : 'Anthropic Organization ID is required'; + }, [values.organization]); + + useEffect(() => { + setErrors({ organization: error }); + }, [error]); + + const handleChange = (e: ChangeEvent) => { + setValues({ organization: e.target.value }); + }; + + return ( + + + {error && {error}} + + ); +}; diff --git a/config-ui/src/plugins/register/claude-code/connection-fields/styled.ts b/config-ui/src/plugins/register/claude-code/connection-fields/styled.ts new file mode 100644 index 00000000000..82ba9f717d0 --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/connection-fields/styled.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import styled from 'styled-components'; + +export const ErrorText = styled.div` + margin-top: 4px; + color: #f5222d; + font-size: 12px; +`; diff --git a/config-ui/src/plugins/register/claude-code/connection-fields/token.tsx b/config-ui/src/plugins/register/claude-code/connection-fields/token.tsx new file mode 100644 index 00000000000..507f04aca20 --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/connection-fields/token.tsx @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo, type ChangeEvent } from 'react'; +import { Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const Token = ({ type, initialValues, values, setValues, setErrors }: Props) => { + useEffect(() => { + setValues({ token: type === 'create' ? initialValues.token ?? '' : undefined }); + }, [type, initialValues.token]); + + const customHeaders: Array<{ key: string; value: string }> = values.customHeaders ?? []; + const hasValidCustomHeaders = customHeaders.some((h) => h.key?.trim() && h.value?.trim()); + const hasIncompleteCustomHeaders = !hasValidCustomHeaders && customHeaders.length > 0; + + const error = useMemo(() => { + if (type === 'update') return ''; + if (hasIncompleteCustomHeaders) + return 'Custom headers are present but none have both a key and a value. Please complete or remove them, or provide an Anthropic API Key.'; + if (hasValidCustomHeaders) return ''; + return values.token?.trim() ? '' : 'Anthropic API Key is required (unless custom headers are configured)'; + }, [type, values.token, hasValidCustomHeaders, hasIncompleteCustomHeaders]); + + useEffect(() => { + setErrors({ token: error }); + }, [error]); + + return ( + + ) => setValues({ token: e.target.value })} + /> + {error && {error}} + + ); +}; diff --git a/config-ui/src/plugins/register/claude-code/index.ts b/config-ui/src/plugins/register/claude-code/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/claude-code/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/register/gh-copilot/assets/icon.svg b/config-ui/src/plugins/register/gh-copilot/assets/icon.svg new file mode 100644 index 00000000000..cb70f7c1d90 --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/gh-copilot/config.tsx b/config-ui/src/plugins/register/gh-copilot/config.tsx new file mode 100644 index 00000000000..5382b0f0d6f --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/config.tsx @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; +import { Organization, Enterprise } from './connection-fields'; + +export const GhCopilotConfig: IPluginConfig = { + plugin: 'gh-copilot', + name: 'GitHub Copilot', + icon: ({ color }) => , + sort: 6.5, + isBeta: true, + connection: { + docLink: 'https://github.com/apache/incubator-devlake/blob/main/backend/plugins/gh-copilot/README.md', + initialValues: { + endpoint: 'https://api.github.com', + organization: '', + enterprise: '', + token: '', + rateLimitPerHour: 5000, + }, + fields: [ + 'name', + 'endpoint', + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + ({ type, initialValues, values, setValues, setErrors }: any) => ( + + ), + { + key: 'token', + label: 'GitHub Personal Access Token', + subLabel: + 'Use a token with access to the organization/enterprise Copilot metrics (for example: manage_billing:copilot, read:enterprise).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: + 'By default, DevLake uses 5,000 requests/hour for GitHub Copilot data collection. Adjust this value to throttle collection speed.', + defaultValue: 5000, + }, + ], + }, + dataScope: { + title: 'Scopes', + }, + scopeConfig: { + entities: ['COPILOT'], + transformation: { + implementationDate: null, + baselinePeriodDays: 90, + }, + }, +}; diff --git a/config-ui/src/plugins/register/gh-copilot/connection-fields/enterprise.tsx b/config-ui/src/plugins/register/gh-copilot/connection-fields/enterprise.tsx new file mode 100644 index 00000000000..d35a1368d19 --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/connection-fields/enterprise.tsx @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo } from 'react'; +import { Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const Enterprise = ({ type, initialValues, values, setValues, setErrors }: Props) => { + useEffect(() => { + setValues({ enterprise: initialValues.enterprise ?? '' }); + }, [initialValues.enterprise]); + + const error = useMemo(() => { + if (type === 'update') return ''; + const hasOrg = !!values.organization?.trim(); + const hasEnt = !!values.enterprise?.trim(); + return hasOrg || hasEnt ? '' : 'At least one of Organization or Enterprise Slug is required'; + }, [type, values.organization, values.enterprise]); + + useEffect(() => { + setErrors({ scopeIdentity: error }); + }, [error]); + + const handleChange = (e: React.ChangeEvent) => { + setValues({ enterprise: e.target.value }); + }; + + return ( + + + {error && {error}} + + ); +}; diff --git a/config-ui/src/plugins/register/gh-copilot/connection-fields/index.ts b/config-ui/src/plugins/register/gh-copilot/connection-fields/index.ts new file mode 100644 index 00000000000..0b6fa90e9e9 --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/connection-fields/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './organization'; +export * from './enterprise'; diff --git a/config-ui/src/plugins/register/gh-copilot/connection-fields/organization.tsx b/config-ui/src/plugins/register/gh-copilot/connection-fields/organization.tsx new file mode 100644 index 00000000000..85f4836020c --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/connection-fields/organization.tsx @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useMemo } from 'react'; +import { Input } from 'antd'; + +import { Block } from '@/components'; + +import * as S from './styled'; + +interface Props { + type: 'create' | 'update'; + initialValues: any; + values: any; + setValues: (value: any) => void; + setErrors: (value: any) => void; +} + +export const Organization = ({ type, initialValues, values, setValues, setErrors }: Props) => { + useEffect(() => { + setValues({ organization: initialValues.organization ?? '' }); + }, [initialValues.organization]); + + const error = useMemo(() => { + if (type === 'update') return ''; + const hasOrg = !!values.organization?.trim(); + const hasEnt = !!values.enterprise?.trim(); + return hasOrg || hasEnt ? '' : 'At least one of Organization or Enterprise Slug is required'; + }, [type, values.organization, values.enterprise]); + + useEffect(() => { + setErrors({ scopeIdentity: error }); + }, [error]); + + const handleChange = (e: React.ChangeEvent) => { + setValues({ organization: e.target.value }); + }; + + return ( + + + {error && {error}} + + ); +}; diff --git a/config-ui/src/plugins/register/gh-copilot/connection-fields/styled.ts b/config-ui/src/plugins/register/gh-copilot/connection-fields/styled.ts new file mode 100644 index 00000000000..b068ff9910b --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/connection-fields/styled.ts @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import styled from 'styled-components'; + +export const ErrorText = styled.div` + margin-top: 4px; + color: ${({ theme }) => theme.colors.error}; + font-size: 12px; +`; diff --git a/config-ui/src/plugins/register/gh-copilot/index.ts b/config-ui/src/plugins/register/gh-copilot/index.ts new file mode 100644 index 00000000000..5f16858cbe4 --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; +export * from './transformation'; diff --git a/config-ui/src/plugins/register/gh-copilot/transformation.tsx b/config-ui/src/plugins/register/gh-copilot/transformation.tsx new file mode 100644 index 00000000000..a715b638828 --- /dev/null +++ b/config-ui/src/plugins/register/gh-copilot/transformation.tsx @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { CaretRightOutlined } from '@ant-design/icons'; +import { theme, Collapse, Tag, DatePicker, InputNumber, Alert } from 'antd'; +import dayjs from 'dayjs'; + +import { HelpTooltip } from '@/components'; + +interface Props { + entities: string[]; + transformation: any; + setTransformation: React.Dispatch>; +} + +export const GhCopilotTransformation = ({ entities, transformation, setTransformation }: Props) => { + const { token } = theme.useToken(); + + const panelStyle: React.CSSProperties = { + marginBottom: 24, + background: token.colorFillAlter, + borderRadius: token.borderRadiusLG, + border: 'none', + }; + + return ( + } + style={{ background: token.colorBgContainer }} + size="large" + items={renderCollapseItems({ + entities, + panelStyle, + transformation, + onChangeTransformation: setTransformation, + token, + })} + /> + ); +}; + +const renderCollapseItems = ({ + entities, + panelStyle, + transformation, + onChangeTransformation, + token, +}: { + entities: string[]; + panelStyle: React.CSSProperties; + transformation: any; + onChangeTransformation: any; + token: any; +}) => + [ + { + key: 'COPILOT', + label: 'GitHub Copilot Impact Analysis', + style: panelStyle, + children: ( + <> +

+ Rollout Milestone Configuration + + Impact Dashboard + +

+ +
+ GitHub Copilot Rollout Date (Optional) + +
+
+ + onChangeTransformation({ + ...transformation, + implementationDate: date ? date.format('YYYY-MM-DD') : null, + }) + } + /> +
+
+ Baseline Period (Days) + +
+
+ + onChangeTransformation({ + ...transformation, + baselinePeriodDays: value ?? 90, + }) + } + /> + + (Recommended: 30-90 days for meaningful comparison) + +
+ + ), + }, + ].filter((it) => entities.includes(it.key) || entities.includes('COPILOT')); diff --git a/config-ui/src/plugins/register/github/connection-fields/styled.ts b/config-ui/src/plugins/register/github/connection-fields/styled.ts index a9522799e0d..96ffc2eb0d7 100644 --- a/config-ui/src/plugins/register/github/connection-fields/styled.ts +++ b/config-ui/src/plugins/register/github/connection-fields/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` @@ -48,11 +48,11 @@ export const Input = styled.div` margin-left: 4px; span.error { - color: #f5222d; + color: ${({ theme }) => theme.colors.error}; } span.success { - color: #4db764; + color: ${({ theme }) => theme.colors.success}; } } } @@ -65,7 +65,7 @@ export const Input = styled.div` export const Alert = styled.div` margin-top: 8px; padding: 12px 20px; - background: #f9f9fa; - border: 1px solid #dbdcdf; + background: ${({ theme }) => theme.colors.bgLayout}; + border: 1px solid ${({ theme }) => theme.colors.border}; border-radius: 4px; `; diff --git a/config-ui/src/plugins/register/github/connection-fields/token.tsx b/config-ui/src/plugins/register/github/connection-fields/token.tsx index 95a322c58b8..4094c672e9a 100644 --- a/config-ui/src/plugins/register/github/connection-fields/token.tsx +++ b/config-ui/src/plugins/register/github/connection-fields/token.tsx @@ -171,9 +171,9 @@ export const Token = ({ {status && (

- {status === 'success' && } - {status === 'warning' && } - {status === 'error' && } + {status === 'success' && } + {status === 'warning' && } + {status === 'error' && } Token Permissions

{status === 'success' &&

All required fields are checked.

} diff --git a/config-ui/src/plugins/register/github/transformation.tsx b/config-ui/src/plugins/register/github/transformation.tsx index ef9d4d7473f..dcb61633ae9 100644 --- a/config-ui/src/plugins/register/github/transformation.tsx +++ b/config-ui/src/plugins/register/github/transformation.tsx @@ -206,7 +206,7 @@ const renderCollapseItems = ({ label={ <> Issue Severity - + } > @@ -277,7 +277,7 @@ const renderCollapseItems = ({ }) } /> - * + *
@@ -368,12 +368,12 @@ const renderCollapseItems = ({ content={ <>
- + Example 1: PR #321 body contains "Closes #1234" (PR #321 and issue #1234 will be mapped by the following RegEx)
- + Example 2: PR #321 body contains "Related to #1234" (PR #321 and issue #1234 will NOT be mapped by the following RegEx)
@@ -453,9 +453,7 @@ const renderCollapseItems = ({ value={(transformation.prSizeExcludedFileExtensions || []).join(',')} onChange={(e) => { // Don't filter during onChange to allow typing commas freely - const extensions = e.target.value - .split(',') - .map((s: string) => s.trim()); + const extensions = e.target.value.split(',').map((s: string) => s.trim()); onChangeTransformation({ ...transformation, prSizeExcludedFileExtensions: extensions, diff --git a/config-ui/src/plugins/register/gitlab/transformation.tsx b/config-ui/src/plugins/register/gitlab/transformation.tsx index 2405e661454..5e324ce7be6 100644 --- a/config-ui/src/plugins/register/gitlab/transformation.tsx +++ b/config-ui/src/plugins/register/gitlab/transformation.tsx @@ -156,7 +156,7 @@ const renderCollapseItems = ({ }) } /> - * + *
diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 7cc69aaed7a..83e7fc670e1 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -19,16 +19,21 @@ import { IPluginConfig } from '@/types'; import { ArgoCDConfig } from './argocd'; +import { AsanaConfig } from './asana'; import { AzureConfig, AzureGoConfig } from './azure'; import { BambooConfig } from './bamboo'; import { BitbucketConfig } from './bitbucket'; import { BitbucketServerConfig } from './bitbucket-server'; import { CircleCIConfig } from './circleci'; +import { ClaudeCodeConfig } from './claude-code'; import { GitHubConfig } from './github'; +import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; +import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; import { TAPDConfig } from './tapd'; import { WebhookConfig } from './webhook'; @@ -39,21 +44,28 @@ import { TeambitionConfig } from './teambition'; import { TestmoConfig } from './testmo'; import { SlackConfig } from './slack/config'; import { CursorConfig } from './cursor'; +import { TempoConfig } from './tempo'; export const pluginConfigs: IPluginConfig[] = [ ArgoCDConfig, + AsanaConfig, AzureConfig, AzureGoConfig, BambooConfig, BitbucketConfig, BitbucketServerConfig, CircleCIConfig, + ClaudeCodeConfig, GitHubConfig, + GhCopilotConfig, GitLabConfig, JenkinsConfig, JiraConfig, + LinearConfig, PagerDutyConfig, + RootlyConfig, SlackConfig, + TempoConfig, QDevConfig, SonarQubeConfig, TAPDConfig, diff --git a/config-ui/src/plugins/register/jenkins/config.tsx b/config-ui/src/plugins/register/jenkins/config.tsx index 68d62f089fd..d007088682e 100644 --- a/config-ui/src/plugins/register/jenkins/config.tsx +++ b/config-ui/src/plugins/register/jenkins/config.tsx @@ -28,6 +28,7 @@ export const JenkinsConfig: IPluginConfig = { sort: 8, connection: { docLink: DOC_URL.PLUGIN.JENKINS.BASIS, + initialValues: {}, fields: [ 'name', { diff --git a/config-ui/src/plugins/register/jenkins/transformation.tsx b/config-ui/src/plugins/register/jenkins/transformation.tsx index 0a44f1dbd44..0cfbf98f43a 100644 --- a/config-ui/src/plugins/register/jenkins/transformation.tsx +++ b/config-ui/src/plugins/register/jenkins/transformation.tsx @@ -100,7 +100,7 @@ const renderCollapseItems = ({ }) } /> - * + *
diff --git a/config-ui/src/plugins/register/jira/config.tsx b/config-ui/src/plugins/register/jira/config.tsx index c6544d6e9e7..1a28f01a749 100644 --- a/config-ui/src/plugins/register/jira/config.tsx +++ b/config-ui/src/plugins/register/jira/config.tsx @@ -29,6 +29,7 @@ export const JiraConfig: IPluginConfig = { sort: 9, connection: { docLink: DOC_URL.PLUGIN.JIRA.BASIS, + initialValues: {}, fields: [ 'name', ({ type, initialValues, values, errors, setValues, setErrors }: any) => ( @@ -49,7 +50,7 @@ export const JiraConfig: IPluginConfig = { 'By default, DevLake uses dynamic rate limit for optimized data collection for Jira. But you can adjust the collection speed by setting up your desirable rate limit.', learnMore: DOC_URL.PLUGIN.JIRA.RATE_LIMIT, externalInfo: - 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server, please contact your admin for more information.', + 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server / Jira Data Center, please contact your admin for more information.', defaultValue: 10000, }, ], @@ -64,6 +65,7 @@ export const JiraConfig: IPluginConfig = { typeMappings: {}, remotelinkCommitShaPattern: '', remotelinkRepoPattern: [], + extraJql: '', }, }, }; diff --git a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx index a85999ba3b0..bc43a6acec3 100644 --- a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx +++ b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx @@ -124,7 +124,7 @@ export const Auth = ({ type, initialValues, values, setValues, setErrors }: Prop Jira Cloud - Jira Server + Jira Server / Jira Data Center } diff --git a/config-ui/src/plugins/register/jira/connection-fields/styled.ts b/config-ui/src/plugins/register/jira/connection-fields/styled.ts index 11e47a3d0a1..acdd9a72879 100644 --- a/config-ui/src/plugins/register/jira/connection-fields/styled.ts +++ b/config-ui/src/plugins/register/jira/connection-fields/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` diff --git a/config-ui/src/plugins/register/jira/transformation-fields/styled.ts b/config-ui/src/plugins/register/jira/transformation-fields/styled.ts index c4363e38386..6b6b2f44512 100644 --- a/config-ui/src/plugins/register/jira/transformation-fields/styled.ts +++ b/config-ui/src/plugins/register/jira/transformation-fields/styled.ts @@ -32,7 +32,7 @@ export const CrossDomain = styled.div` span { padding: 4px 8px; - background-color: #efefef; + background-color: ${({ theme }) => theme.colors.bgMuted}; } span + span { @@ -53,7 +53,7 @@ export const RemoteLinkWrapper = styled.div` .error { margin-top: 2px; - color: #cd4246; + color: ${({ theme }) => theme.colors.errorMuted}; } `; @@ -63,7 +63,7 @@ export const DialogBody = styled.div` padding: 8px 16px; max-height: 240px; overflow-y: auto; - background: #efefef; + background: ${({ theme }) => theme.colors.bgMuted}; } .search { diff --git a/config-ui/src/plugins/register/jira/transformation.tsx b/config-ui/src/plugins/register/jira/transformation.tsx index 97cd20967b5..bbfe760dba0 100644 --- a/config-ui/src/plugins/register/jira/transformation.tsx +++ b/config-ui/src/plugins/register/jira/transformation.tsx @@ -19,7 +19,7 @@ import { useState, useEffect } from 'react'; import { uniqWith } from 'lodash'; import { CaretRightOutlined } from '@ant-design/icons'; -import { theme, Collapse, Tag, Form, Select } from 'antd'; +import { theme, Collapse, Tag, Form, Select, Input } from 'antd'; import API from '@/api'; import { PageLoading, HelpTooltip, ExternalLink } from '@/components'; @@ -265,6 +265,26 @@ const renderCollapseItems = ({ } /> + + Extra JQL + + + } + extra='Tip: use Go template variables to make this dynamic, e.g. owner = "{{.BoardName}}"' + > + + onChangeTransformation({ + ...transformation, + extraJql: e.target.value, + }) + } + /> + ), }, diff --git a/config-ui/src/plugins/register/linear/assets/icon.svg b/config-ui/src/plugins/register/linear/assets/icon.svg new file mode 100644 index 00000000000..64cddfdea54 --- /dev/null +++ b/config-ui/src/plugins/register/linear/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/linear/config.tsx b/config-ui/src/plugins/register/linear/config.tsx new file mode 100644 index 00000000000..28cf9f98a95 --- /dev/null +++ b/config-ui/src/plugins/register/linear/config.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const LinearConfig: IPluginConfig = { + plugin: 'linear', + name: 'Linear', + icon: ({ color }) => , + sort: 13, + connection: { + docLink: 'https://developers.linear.app/docs', + initialValues: { + endpoint: 'https://api.linear.app/graphql', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Linear GraphQL API base URL.', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'Your Linear personal API key (Settings → Security & access → Personal API keys).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (1500).', + defaultValue: 1500, + }, + ], + }, + dataScope: { + title: 'Teams', + searchPlaceholder: 'Search teams...', + }, +}; diff --git a/config-ui/src/plugins/register/linear/index.ts b/config-ui/src/plugins/register/linear/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/linear/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/register/opsgenie/connection-fields/styled.ts b/config-ui/src/plugins/register/opsgenie/connection-fields/styled.ts index a9522799e0d..96ffc2eb0d7 100644 --- a/config-ui/src/plugins/register/opsgenie/connection-fields/styled.ts +++ b/config-ui/src/plugins/register/opsgenie/connection-fields/styled.ts @@ -24,7 +24,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` @@ -48,11 +48,11 @@ export const Input = styled.div` margin-left: 4px; span.error { - color: #f5222d; + color: ${({ theme }) => theme.colors.error}; } span.success { - color: #4db764; + color: ${({ theme }) => theme.colors.success}; } } } @@ -65,7 +65,7 @@ export const Input = styled.div` export const Alert = styled.div` margin-top: 8px; padding: 12px 20px; - background: #f9f9fa; - border: 1px solid #dbdcdf; + background: ${({ theme }) => theme.colors.bgLayout}; + border: 1px solid ${({ theme }) => theme.colors.border}; border-radius: 4px; `; diff --git a/config-ui/src/plugins/register/q-dev/connection-fields/aws-credentials.tsx b/config-ui/src/plugins/register/q-dev/connection-fields/aws-credentials.tsx index 19953d4459b..405d33806fd 100644 --- a/config-ui/src/plugins/register/q-dev/connection-fields/aws-credentials.tsx +++ b/config-ui/src/plugins/register/q-dev/connection-fields/aws-credentials.tsx @@ -51,7 +51,7 @@ export const AwsCredentials = ({ type, initialValues, values, setValues, setErro const accessKeyId = values.accessKeyId ?? ''; const secretAccessKey = values.secretAccessKey ?? ''; const region = values.region ?? ''; - + const isAccessKeyAuth = authType === 'access_key'; useEffect(() => { @@ -141,13 +141,13 @@ export const AwsCredentials = ({ type, initialValues, values, setValues, setErro const handleAuthTypeChange = (e: any) => { const newAuthType = e.target.value; setValues({ authType: newAuthType }); - + // Clear access key fields when switching to IAM role if (newAuthType === 'iam_role') { - setValues({ + setValues({ authType: newAuthType, accessKeyId: '', - secretAccessKey: '' + secretAccessKey: '', }); } }; @@ -163,7 +163,11 @@ export const AwsCredentials = ({ type, initialValues, values, setValues, setErro {isAccessKeyAuth && ( <> - + - {accessKeyError &&
{accessKeyError}
} + {accessKeyError &&
{accessKeyError}
}
- + - {secretKeyError &&
{secretKeyError}
} + {secretKeyError &&
{secretKeyError}
}
)} {!isAccessKeyAuth && ( - -
+ +

- Make sure the IAM role has the necessary S3 permissions to access your bucket. - No additional credentials are required when using IAM role authentication. + Make sure the IAM role has the necessary S3 permissions to access your bucket. No additional credentials + are required when using IAM role authentication.

@@ -206,7 +224,7 @@ export const AwsCredentials = ({ type, initialValues, values, setValues, setErro onChange={handleRegionChange} status={regionError ? 'error' : ''} /> - {regionError &&
{regionError}
} + {regionError &&
{regionError}
} ); diff --git a/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx b/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx index bc0eb73398d..5157338fc45 100644 --- a/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx +++ b/config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx @@ -56,12 +56,18 @@ export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues return API.connection.test(plugin, connectionId, { authType: values.authType !== initialValues.authType ? values.authType : undefined, accessKeyId: values.accessKeyId !== initialValues.accessKeyId ? values.accessKeyId : undefined, - secretAccessKey: values.secretAccessKey !== initialValues.secretAccessKey ? values.secretAccessKey : undefined, + secretAccessKey: + values.secretAccessKey !== initialValues.secretAccessKey ? values.secretAccessKey : undefined, region: values.region !== initialValues.region ? values.region : undefined, bucket: values.bucket !== initialValues.bucket ? values.bucket : undefined, - identityStoreId: values.identityStoreId !== initialValues.identityStoreId ? values.identityStoreId : undefined, - identityStoreRegion: values.identityStoreRegion !== initialValues.identityStoreRegion ? values.identityStoreRegion : undefined, - rateLimitPerHour: values.rateLimitPerHour !== initialValues.rateLimitPerHour ? values.rateLimitPerHour : undefined, + identityStoreId: + values.identityStoreId !== initialValues.identityStoreId ? values.identityStoreId : undefined, + identityStoreRegion: + values.identityStoreRegion !== initialValues.identityStoreRegion + ? values.identityStoreRegion + : undefined, + rateLimitPerHour: + values.rateLimitPerHour !== initialValues.rateLimitPerHour ? values.rateLimitPerHour : undefined, proxy: values.proxy !== initialValues.proxy ? values.proxy : undefined, } as any); } else { @@ -104,7 +110,7 @@ export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues } } catch (error: any) { let errorMessage = 'Connection test failed. Please check your configuration.'; - + if (error?.response?.data?.message) { errorMessage = error.response.data.message; } else if (error?.message) { @@ -121,7 +127,8 @@ export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues } else if (errorMessage.includes('InvalidBucketName')) { errorMessage = 'Invalid S3 bucket name. Please check the bucket name format.'; } else if (errorMessage.includes('NoCredentialsError')) { - errorMessage = 'AWS credentials not found. Please provide valid Access Key ID and Secret Access Key, or ensure IAM role is properly configured.'; + errorMessage = + 'AWS credentials not found. Please provide valid Access Key ID and Secret Access Key, or ensure IAM role is properly configured.'; } setTestResult({ @@ -165,11 +172,9 @@ export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues testResult?.success && testResult.details ? (
✓ S3 Access: Verified
- {testResult.details.identityCenterAccess && ( -
✓ IAM Identity Center: Configured
- )} + {testResult.details.identityCenterAccess &&
✓ IAM Identity Center: Configured
} {!values.identityStoreId && ( -
+
⚠️ IAM Identity Center not configured - user display names will show as user IDs
)} @@ -182,4 +187,4 @@ export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues )} ); -}; \ No newline at end of file +}; diff --git a/config-ui/src/plugins/register/q-dev/connection-fields/identity-center-config.tsx b/config-ui/src/plugins/register/q-dev/connection-fields/identity-center-config.tsx index bd2333807cb..b467a32013d 100644 --- a/config-ui/src/plugins/register/q-dev/connection-fields/identity-center-config.tsx +++ b/config-ui/src/plugins/register/q-dev/connection-fields/identity-center-config.tsx @@ -105,7 +105,7 @@ export const IdentityCenterConfig = ({ initialValues, values, setValues, setErro onChange={handleStoreIdChange} status={storeIdError ? 'error' : ''} /> - {storeIdError &&
{storeIdError}
} + {storeIdError &&
{storeIdError}
} - {regionError &&
{regionError}
} + {regionError &&
{regionError}
}
); diff --git a/config-ui/src/plugins/register/q-dev/connection-fields/s3-config.tsx b/config-ui/src/plugins/register/q-dev/connection-fields/s3-config.tsx index d36843080df..734801fdbad 100644 --- a/config-ui/src/plugins/register/q-dev/connection-fields/s3-config.tsx +++ b/config-ui/src/plugins/register/q-dev/connection-fields/s3-config.tsx @@ -70,7 +70,7 @@ export const S3Config = ({ initialValues, values, setValues, setErrors }: Props) onChange={handleBucketChange} status={bucketError ? 'error' : ''} /> - {bucketError &&
{bucketError}
} + {bucketError &&
{bucketError}
} ); }; diff --git a/config-ui/src/plugins/register/q-dev/data-scope.tsx b/config-ui/src/plugins/register/q-dev/data-scope.tsx index f6aa5a9b0ef..657d0bdb890 100644 --- a/config-ui/src/plugins/register/q-dev/data-scope.tsx +++ b/config-ui/src/plugins/register/q-dev/data-scope.tsx @@ -26,6 +26,7 @@ interface ScopeData { year?: number; month?: number | null; basePath?: string; + accountId?: string; } interface ScopeItem { @@ -46,7 +47,7 @@ const CURRENT_YEAR = new Date().getUTCFullYear(); const MONTHS = Array.from({ length: 12 }, (_, idx) => idx + 1); const MONTH_LABELS = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; -const DEFAULT_BASE_PATH = 'user-report/AWSLogs'; +const DEFAULT_BASE_PATH = 'user-report'; const ensureLeadingZero = (value: number) => value.toString().padStart(2, '0'); @@ -92,11 +93,14 @@ const extractScopeMeta = (item: ScopeItem) => { const basePath = normalizeBasePath(data.basePath ?? (baseSegments.length ? baseSegments.join('/') : '')); + const accountId = data.accountId ?? ''; + return { basePath, year: typeof year === 'number' ? year : null, month, prefix, + accountId, }; }; @@ -110,12 +114,25 @@ const deriveBasePathFromSelection = (items: ScopeItem[]) => { return undefined; }; -const buildPrefix = (basePath: string, year: number, month: number | null) => { +const deriveAccountIdFromSelection = (items: ScopeItem[]) => { + for (const item of items) { + const meta = extractScopeMeta(item); + if (meta.accountId) { + return meta.accountId; + } + } + return undefined; +}; + +const buildPrefix = (basePath: string, year: number, month: number | null, accountId?: string) => { const segments = [] as string[]; const sanitizedBase = normalizeBasePath(basePath); if (sanitizedBase) { segments.push(sanitizedBase); } + if (accountId) { + segments.push(accountId); + } segments.push(String(year)); if (month !== null && month !== undefined) { segments.push(ensureLeadingZero(month)); @@ -123,13 +140,14 @@ const buildPrefix = (basePath: string, year: number, month: number | null) => { return segments.join('/'); }; -const createScopeItem = (basePath: string, year: number, month: number | null): ScopeItem => { +const createScopeItem = (basePath: string, year: number, month: number | null, accountId?: string): ScopeItem => { const sanitizedBase = normalizeBasePath(basePath); - const prefix = buildPrefix(sanitizedBase, year, month); + const prefix = buildPrefix(sanitizedBase, year, month, accountId); const isFullYear = month === null; - const name = isFullYear + const timeLabel = isFullYear ? `${year} (Full Year)` : `${year}-${ensureLeadingZero(month as number)} (${MONTH_LABELS[(month as number) - 1]})`; + const name = accountId ? `${accountId} ${timeLabel}` : timeLabel; return { id: prefix, @@ -137,6 +155,7 @@ const createScopeItem = (basePath: string, year: number, month: number | null): fullName: prefix, data: { basePath: sanitizedBase, + accountId: accountId || undefined, prefix, year, month, @@ -165,6 +184,7 @@ const MONTH_OPTIONS = MONTHS.map((value) => ({ type FormValues = { basePath: string; + accountId: string; year: number; mode: 'year' | 'months'; months?: number[]; @@ -185,20 +205,29 @@ export const QDevDataScope = ({ [selectedItems], ); + const derivedAccountId = useMemo(() => deriveAccountIdFromSelection(selectedItems) ?? '', [selectedItems]); + useEffect(() => { if (!form.isFieldsTouched(['basePath'])) { form.setFieldsValue({ basePath: derivedBasePath }); } }, [derivedBasePath, form]); + useEffect(() => { + if (!form.isFieldsTouched(['accountId'])) { + form.setFieldsValue({ accountId: derivedAccountId }); + } + }, [derivedAccountId, form]); + useEffect(() => { form.setFieldsValue({ mode: 'year', year: form.getFieldValue('year') ?? CURRENT_YEAR }); }, [form]); const handleAdd = async () => { - const { basePath, year, mode, months = [] } = await form.validateFields(); + const { basePath, accountId, year, mode, months = [] } = await form.validateFields(); const normalizedBase = normalizeBasePath(basePath ?? ''); + const normalizedAccountId = (accountId ?? '').trim(); const normalizedYear = Number(year); if (!normalizedYear || Number.isNaN(normalizedYear)) { return; @@ -209,6 +238,7 @@ export const QDevDataScope = ({ const meta = extractScopeMeta(item); return ( meta.basePath === normalizedBase && + meta.accountId === normalizedAccountId && meta.year === normalizedYear && (meta.month === null || meta.month === undefined) ); @@ -223,14 +253,19 @@ export const QDevDataScope = ({ const hasMonths = selectedItems.some((item) => { const meta = extractScopeMeta(item); - return meta.basePath === normalizedBase && meta.year === normalizedYear && meta.month !== null; + return ( + meta.basePath === normalizedBase && + meta.accountId === normalizedAccountId && + meta.year === normalizedYear && + meta.month !== null + ); }); if (hasMonths) { return; } - const item = createScopeItem(normalizedBase, normalizedYear, null); + const item = createScopeItem(normalizedBase, normalizedYear, null, normalizedAccountId || undefined); if (!currentIds.has(item.id) && !disabledIds.has(item.id)) { additions.push(item); } @@ -249,7 +284,7 @@ export const QDevDataScope = ({ return; } - const item = createScopeItem(normalizedBase, normalizedYear, month); + const item = createScopeItem(normalizedBase, normalizedYear, month, normalizedAccountId || undefined); if (currentIds.has(item.id) || disabledIds.has(item.id)) { return; } @@ -282,24 +317,36 @@ export const QDevDataScope = ({ render: (_: unknown, item) => formatScopeLabel(item), }, { - title: 'S3 Prefix', + title: 'Scope Path', dataIndex: 'id', key: 'prefix', render: (_: unknown, item) => { const meta = extractScopeMeta(item); + if (meta.accountId) { + const timePart = meta.month ? `${meta.year}/${ensureLeadingZero(meta.month)}` : `${meta.year}`; + return ( + + + {meta.basePath}/…/{meta.accountId}/…/{timePart} + + + ); + } return {meta.prefix}; }, }, { - title: 'Base Path', + title: 'Account ID', dataIndex: 'id', - key: 'basePath', + key: 'accountId', render: (_: unknown, item) => { const meta = extractScopeMeta(item); - return meta.basePath ? ( - {meta.basePath} + return meta.accountId ? ( + {meta.accountId} ) : ( - (bucket root) + ); }, }, @@ -335,6 +382,7 @@ export const QDevDataScope = ({ layout="inline" initialValues={{ basePath: derivedBasePath, + accountId: derivedAccountId, year: CURRENT_YEAR, mode: 'year', months: [], @@ -346,9 +394,18 @@ export const QDevDataScope = ({ label="Base Path" name="basePath" style={{ flex: 1 }} - tooltip="Common prefix in S3 between the bucket root and the year directory" + tooltip="S3 prefix before the AWSLogs directory (e.g. 'user-report')" + > + + + + - + diff --git a/config-ui/src/plugins/register/rootly/assets/icon.svg b/config-ui/src/plugins/register/rootly/assets/icon.svg new file mode 100644 index 00000000000..319bb6d007e --- /dev/null +++ b/config-ui/src/plugins/register/rootly/assets/icon.svg @@ -0,0 +1,28 @@ + + + + + + + + + + + + + diff --git a/config-ui/src/plugins/register/rootly/config.tsx b/config-ui/src/plugins/register/rootly/config.tsx new file mode 100644 index 00000000000..163852d04cd --- /dev/null +++ b/config-ui/src/plugins/register/rootly/config.tsx @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { DOC_URL } from '@/release'; +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const RootlyConfig: IPluginConfig = { + plugin: 'rootly', + name: 'Rootly', + icon: ({ color }) => , + sort: 16, + isBeta: true, + connection: { + docLink: DOC_URL.PLUGIN.ROOTLY.BASIS, + initialValues: { + endpoint: 'https://api.rootly.com/v1/', + }, + fields: [ + 'name', + { + key: 'endpoint', + multipleVersions: { + cloud: 'https://api.rootly.com/v1/', + server: '', + }, + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: + 'By default, DevLake uses 3,600 requests/hour for data collection for Rootly. But you can adjust the collection speed by setting up your desirable rate limit.', + learnMore: DOC_URL.PLUGIN.ROOTLY.RATE_LIMIT, + defaultValue: 3600, + }, + ], + }, + dataScope: { + title: 'Services', + }, +}; diff --git a/config-ui/src/plugins/register/rootly/index.ts b/config-ui/src/plugins/register/rootly/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/rootly/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/register/sonarqube/config.tsx b/config-ui/src/plugins/register/sonarqube/config.tsx index 5da394d2096..966ce8bf5af 100644 --- a/config-ui/src/plugins/register/sonarqube/config.tsx +++ b/config-ui/src/plugins/register/sonarqube/config.tsx @@ -55,7 +55,11 @@ export const SonarQubeConfig: IPluginConfig = { setErrors={setErrors} /> ), - 'token', + { + key: 'token', + subLabel: + 'A token with Browse permission on the projects you want is sufficient for listing projects and collecting issues, hotspots, and file metrics. Listing Accounts (users) may require a system admin token on some SonarQube instances.', + }, 'proxy', { key: 'rateLimitPerHour', diff --git a/config-ui/src/plugins/register/sonarqube/connection-fields/organization.tsx b/config-ui/src/plugins/register/sonarqube/connection-fields/organization.tsx index 8cd71f61a38..89cf00a04ce 100644 --- a/config-ui/src/plugins/register/sonarqube/connection-fields/organization.tsx +++ b/config-ui/src/plugins/register/sonarqube/connection-fields/organization.tsx @@ -39,7 +39,7 @@ export const Organization = ({ initialValues, values, setValues, setErrors }: Pr useEffect(() => { setErrors({ - org: (values.endpoint != 'https://sonarcloud.io/api/' || values.org) ? '' : 'organization is required', + org: values.endpoint != 'https://sonarcloud.io/api/' || values.org ? '' : 'organization is required', }); }, [values.org]); diff --git a/config-ui/src/plugins/register/tempo/assets/icon.png b/config-ui/src/plugins/register/tempo/assets/icon.png new file mode 100644 index 00000000000..80f4e36f897 Binary files /dev/null and b/config-ui/src/plugins/register/tempo/assets/icon.png differ diff --git a/config-ui/src/plugins/register/tempo/assets/icon.svg b/config-ui/src/plugins/register/tempo/assets/icon.svg new file mode 100644 index 00000000000..6c7407b2b4f --- /dev/null +++ b/config-ui/src/plugins/register/tempo/assets/icon.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/config-ui/src/plugins/register/tempo/config.tsx b/config-ui/src/plugins/register/tempo/config.tsx new file mode 100644 index 00000000000..c4494d13908 --- /dev/null +++ b/config-ui/src/plugins/register/tempo/config.tsx @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { DOC_URL } from '@/release'; +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.png'; + +export const TempoConfig: IPluginConfig = { + plugin: 'tempo', + name: 'Tempo', + icon: () => , + sort: 20, + connection: { + docLink: 'https://devlake.apache.org/docs/Configuration/Tempo', + initialValues: { + endpoint: 'https://api.tempo.io/4', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'REST Endpoint', + subLabel: 'Tempo API v4 base URL', + placeholder: 'https://api.tempo.io/4', + }, + 'token', + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for default.', + defaultValue: 1000, + }, + ], + }, + dataScope: { + title: 'Teams', + millerColumn: { + columnCount: 2.5, + }, + }, + scopeConfig: { + entities: ['TICKET'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/tempo/index.ts b/config-ui/src/plugins/register/tempo/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/tempo/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/register/testmo/config.tsx b/config-ui/src/plugins/register/testmo/config.tsx index 378ef4fa7ad..8b8f7d15788 100644 --- a/config-ui/src/plugins/register/testmo/config.tsx +++ b/config-ui/src/plugins/register/testmo/config.tsx @@ -42,14 +42,12 @@ export const TestmoConfig: IPluginConfig = { label: 'API Token', type: 'password', placeholder: 'Enter your Testmo API token', - subLabel: - 'Generate an API token from your Testmo account settings: Settings → API', + subLabel: 'Generate an API token from your Testmo account settings: Settings → API', }, 'proxy', { key: 'rateLimitPerHour', - subLabel: - 'By default, DevLake will not limit API requests per hour. But you can set a number if you want to.', + subLabel: 'By default, DevLake will not limit API requests per hour. But you can set a number if you want to.', learnMore: 'https://devlake.apache.org/docs/Configuration/Testmo/#rate-limit-api-requests-per-hour', externalInfo: 'Testmo does not specify a maximum number of requests per hour.', defaultValue: 10000, diff --git a/config-ui/src/plugins/register/webhook/components/create-dialog.tsx b/config-ui/src/plugins/register/webhook/components/create-dialog.tsx index bc08a0750d9..3157b6585a2 100644 --- a/config-ui/src/plugins/register/webhook/components/create-dialog.tsx +++ b/config-ui/src/plugins/register/webhook/components/create-dialog.tsx @@ -56,7 +56,13 @@ export const CreateDialog = ({ open, onCancel, onSubmitAfter }: Props) => { const [success, res] = await operator( async () => { const { - webhook: { id, postIssuesEndpoint, closeIssuesEndpoint, postPipelineDeployTaskEndpoint, postPullRequestsEndpoint }, + webhook: { + id, + postIssuesEndpoint, + closeIssuesEndpoint, + postPipelineDeployTaskEndpoint, + postPullRequestsEndpoint, + }, apiKey, } = await dispatch(addWebhook({ name })).unwrap(); diff --git a/config-ui/src/plugins/register/webhook/components/utils.ts b/config-ui/src/plugins/register/webhook/components/utils.ts index ba2fe78bb9c..8675be979ce 100644 --- a/config-ui/src/plugins/register/webhook/components/utils.ts +++ b/config-ui/src/plugins/register/webhook/components/utils.ts @@ -20,8 +20,9 @@ import { IWebhook } from '@/types'; export const transformURI = (prefix: string, webhook: IWebhook, apiKey: string) => { return { - postIssuesEndpoint: `curl ${prefix}${webhook.postIssuesEndpoint} -X 'POST' -H 'Authorization: Bearer ${apiKey ?? '{API_KEY}' - }' -d '{ + postIssuesEndpoint: `curl ${prefix}${webhook.postIssuesEndpoint} -X 'POST' -H 'Authorization: Bearer ${ + apiKey ?? '{API_KEY}' + }' -d '{ "issueKey":"DLK-1234", "title":"an incident from DLK", "type":"INCIDENT", @@ -30,10 +31,12 @@ export const transformURI = (prefix: string, webhook: IWebhook, apiKey: string) "createdDate":"2020-01-01T12:00:00+00:00", "updatedDate":"2020-01-01T12:00:00+00:00" }'`, - closeIssuesEndpoint: `curl ${prefix}${webhook.closeIssuesEndpoint} -X 'POST' -H 'Authorization: Bearer ${apiKey ?? '{API_KEY}' - }'`, - postDeploymentsCurl: `curl ${prefix}${webhook.postPipelineDeployTaskEndpoint} -X 'POST' -H 'Authorization: Bearer ${apiKey ?? '{API_KEY}' - }' -d '{ + closeIssuesEndpoint: `curl ${prefix}${webhook.closeIssuesEndpoint} -X 'POST' -H 'Authorization: Bearer ${ + apiKey ?? '{API_KEY}' + }'`, + postDeploymentsCurl: `curl ${prefix}${webhook.postPipelineDeployTaskEndpoint} -X 'POST' -H 'Authorization: Bearer ${ + apiKey ?? '{API_KEY}' + }' -d '{ "id": "Required. This will be the unique ID of the deployment", "startedDate": "2023-01-01T12:00:00+00:00", "finishedDate": "2023-01-01T12:00:00+00:00", @@ -49,8 +52,9 @@ export const transformURI = (prefix: string, webhook: IWebhook, apiKey: string) } ] }'`, - postPullRequestsEndpoint: `curl ${prefix}${webhook.postPullRequestsEndpoint} -X 'POST' -H 'Authorization: Bearer ${apiKey ?? '{API_KEY}' - }' -d '{ + postPullRequestsEndpoint: `curl ${prefix}${webhook.postPullRequestsEndpoint} -X 'POST' -H 'Authorization: Bearer ${ + apiKey ?? '{API_KEY}' + }' -d '{ "id": "Required. This will be the unique ID of the pull request", "baseRepoId": "your-repo-id", "headRepoId": "your-repo-id", diff --git a/config-ui/src/plugins/register/webhook/styled.ts b/config-ui/src/plugins/register/webhook/styled.ts index ca59c72c150..c7186d44af6 100644 --- a/config-ui/src/plugins/register/webhook/styled.ts +++ b/config-ui/src/plugins/register/webhook/styled.ts @@ -27,7 +27,7 @@ export const Wrapper = styled.div` margin-bottom: 16px; padding: 0; font-weight: 600; - color: #4db764; + color: ${({ theme }) => theme.colors.success}; } h5 { @@ -35,12 +35,12 @@ export const Wrapper = styled.div` } p { - color: #292b3f; + color: ${({ theme }) => theme.colors.text}; } `; export const Action = styled.div` - color: #7497f7; + color: ${({ theme }) => theme.colors.primary}; span { cursor: pointer; diff --git a/config-ui/src/plugins/register/zentao/config.tsx b/config-ui/src/plugins/register/zentao/config.tsx index 5cc8be795bc..e37468a5040 100644 --- a/config-ui/src/plugins/register/zentao/config.tsx +++ b/config-ui/src/plugins/register/zentao/config.tsx @@ -29,6 +29,7 @@ export const ZenTaoConfig: IPluginConfig = { sort: 18, connection: { docLink: DOC_URL.PLUGIN.ZENTAO.BASIS, + initialValues: {}, fields: [ 'name', { diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index c008f4b4da1..34c6ed63b14 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -16,7 +16,9 @@ * */ -import PluginIcon from '@/images/plugin-icon.svg'; +import React from 'react'; + +import PluginIcon from '@/images/plugin-icon.svg?react'; import { pluginConfigs } from './register'; import { IPluginConfig } from '@/types'; @@ -41,20 +43,101 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.planKey}`; case 'argocd': return `${scope.name}`; + case 'asana': + return `${scope.gid}`; + case 'linear': + return `${scope.teamId}`; default: return `${scope.id}`; } }; -export const getRegisterPlugins = () => pluginConfigs.map((it) => it.plugin); +export const getPluginScopeName = (plugin: string, scope: any) => { + if (!scope) { + return ''; + } + + if (plugin === 'claude_code') { + const scopeData = scope.data ?? scope; + + return `${scopeData.organization ?? scope.organization ?? scope.fullName ?? scope.name ?? scope.id ?? ''}`.trim(); + } + + if (plugin === 'gh-copilot') { + const scopeData = scope.data ?? scope; + + const rawId = `${scope.id ?? scopeData.id ?? scope.fullName ?? scope.name ?? ''}`.trim(); + const enterprise = `${scopeData.enterprise ?? ''}`.trim(); + const organization = `${scopeData.organization ?? ''}`.trim(); + + if (enterprise && organization) { + return `${rawId} (Enterprise + Organization)`; + } + if (enterprise) { + return `${rawId} (Enterprise)`; + } + if (organization) { + return `${rawId} (Organization)`; + } + if (rawId.includes('/')) { + return `${rawId} (Enterprise + Organization)`; + } + return rawId; + } + + return `${scope.fullName ?? scope.name ?? scope.id ?? ''}`; +}; + +const pluginAliasMap: Record = { + claude_code: 'claude_code', + copilot: 'gh-copilot', +}; + +const aliasByTarget = Object.entries(pluginAliasMap).reduce>((acc, [alias, target]) => { + if (!acc[target]) { + acc[target] = []; + } + acc[target].push(alias); + return acc; +}, {}); + +export const getRegisterPlugins = () => { + const ordered: string[] = []; + const seen = new Set(); + + for (const config of pluginConfigs) { + if (!seen.has(config.plugin)) { + ordered.push(config.plugin); + seen.add(config.plugin); + } + + for (const alias of aliasByTarget[config.plugin] ?? []) { + if (!seen.has(alias)) { + ordered.push(alias); + seen.add(alias); + } + } + } + + for (const alias of Object.keys(pluginAliasMap)) { + if (!seen.has(alias)) { + ordered.push(alias); + } + } + + return ordered; +}; export const getPluginConfig = (name: string): IPluginConfig => { let pluginConfig = pluginConfigs.find((it) => it.plugin === name); + if (!pluginConfig && pluginAliasMap[name]) { + pluginConfig = pluginConfigs.find((it) => it.plugin === pluginAliasMap[name]); + } if (!pluginConfig) { pluginConfig = { plugin: name, name: name, - icon: PluginIcon, + icon: ({ color }) => React.createElement(PluginIcon, { fill: color }), sort: 101, connection: { docLink: '', diff --git a/config-ui/src/release/stable.ts b/config-ui/src/release/stable.ts index c31162fdb85..118bd544f58 100644 --- a/config-ui/src/release/stable.ts +++ b/config-ui/src/release/stable.ts @@ -23,9 +23,14 @@ const URLS = { }, DORA: 'https://devlake.apache.org/docs/DORA/', PLUGIN: { + ASANA: { + BASIS: 'https://devlake.apache.org/docs/Configuration/Asana', + TRANSFORMATION: 'https://devlake.apache.org/docs/Configuration/Asana', + }, ARGOCD: { BASIS: 'https://devlake.apache.org/docs/Configuration/ArgoCD', - TRANSFORMATION: 'https://devlake.apache.org/docs/Configuration/ArgoCD#step-3---adding-transformation-rules-optional', + TRANSFORMATION: + 'https://devlake.apache.org/docs/Configuration/ArgoCD#step-3---adding-transformation-rules-optional', }, AZUREDEVOPS: { BASIS: 'https://devlake.apache.org/docs/Configuration/AzureDevOps', @@ -102,6 +107,10 @@ const URLS = { BASIS: 'https://devlake.apache.org/docs/Configuration/PagerDuty', RATE_LIMIT: 'https://devlake.apache.org/docs/Configuration/PagerDuty/#custom-rate-limit-optional', }, + ROOTLY: { + BASIS: 'https://devlake.apache.org/docs/Configuration/Rootly', + RATE_LIMIT: 'https://devlake.apache.org/docs/Configuration/Rootly#fixed-rate-limit-optional', + }, SLACK: { BASIS: 'https://devlake.apache.org/docs/Configuration/Slack', RATE_LIMIT: 'https://devlake.apache.org/docs/Configuration/Slack#custom-rate-limit-optional', diff --git a/config-ui/src/routes/blueprint/connection-detail/table.tsx b/config-ui/src/routes/blueprint/connection-detail/table.tsx index 7036d59a233..e2c77a67f7e 100644 --- a/config-ui/src/routes/blueprint/connection-detail/table.tsx +++ b/config-ui/src/routes/blueprint/connection-detail/table.tsx @@ -21,7 +21,7 @@ import { Table } from 'antd'; import API from '@/api'; import { useRefreshData } from '@/hooks'; -import { getPluginScopeId, ScopeConfig } from '@/plugins'; +import { getPluginScopeId, getPluginScopeName, ScopeConfig } from '@/plugins'; interface Props { plugin: string; @@ -36,7 +36,7 @@ export const BlueprintConnectionDetailTable = ({ plugin, connectionId, scopeIds const scopes = await Promise.all(scopeIds.map((scopeId) => API.scope.get(plugin, connectionId, scopeId))); return scopes.map((sc) => ({ id: getPluginScopeId(plugin, sc.scope), - name: sc.scope.fullName ?? sc.scope.name, + name: getPluginScopeName(plugin, sc.scope) || sc.scope.fullName || sc.scope.name, scopeConfigId: sc.scopeConfig?.id, scopeConfigName: sc.scopeConfig?.name, })); diff --git a/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx b/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx index 52491bac936..7a211f239af 100644 --- a/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx +++ b/config-ui/src/routes/blueprint/detail/blueprint-detail.tsx @@ -48,15 +48,12 @@ export const BlueprintDetail = ({ id, from }: Props) => { }, [state]); const { ready, data, error } = useRefreshData(async () => { - const [bpRes, pipelineRes] = await Promise.all([ - API.blueprint.get(id), - API.blueprint.pipelines(id), - ]); + const [bpRes, pipelineRes] = await Promise.all([API.blueprint.get(id), API.blueprint.pipelines(id)]); return [bpRes, pipelineRes.pipelines[0]]; }, [version]); useEffect(() => { - if (axios.isAxiosError(error) && error.response?.status === 404) { + if (axios.isAxiosError(error) && error.response?.status === 404) { message.error(`Blueprint not found with id: ${id}`); navigate(PATHS.BLUEPRINTS(), { replace: true }); } diff --git a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts index 6b3117e2df0..c09a3a22847 100644 --- a/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts +++ b/config-ui/src/routes/blueprint/detail/components/advanced-editor/example/dbt.ts @@ -21,7 +21,7 @@ const dbt = [ { plugin: 'dbt', options: { - projectPath: '/var/www/html/my-project', + projectPath: '/tmp/devlake-dbt-projects/my-project', projectGitURL: '', projectName: 'myproject', projectTarget: 'dev', diff --git a/config-ui/src/routes/blueprint/detail/components/sync-policy/index.tsx b/config-ui/src/routes/blueprint/detail/components/sync-policy/index.tsx index 9efd012f16f..9f7b24c7340 100644 --- a/config-ui/src/routes/blueprint/detail/components/sync-policy/index.tsx +++ b/config-ui/src/routes/blueprint/detail/components/sync-policy/index.tsx @@ -76,7 +76,7 @@ export const SyncPolicy = ({ const cron = useMemo(() => getCron(isManual, cronConfig), [isManual, cronConfig]); - const [mintue, hour, day, month, week] = useMemo(() => cronConfig.split(' '), [cronConfig]); + const [minute, hour, day, month, week] = useMemo(() => cronConfig.split(' '), [cronConfig]); const handleChangeFrequency = (e: RadioChangeEvent) => { const value = e.target.value; @@ -152,32 +152,32 @@ export const SyncPolicy = ({ onChangeCronConfig([e.target.value, hour, day, month, week].join(' '))} /> onChangeCronConfig([mintue, e.target.value, day, month, week].join(' '))} + onChange={(e) => onChangeCronConfig([minute, e.target.value, day, month, week].join(' '))} /> onChangeCronConfig([mintue, hour, e.target.value, month, week].join(' '))} + onChange={(e) => onChangeCronConfig([minute, hour, e.target.value, month, week].join(' '))} /> onChangeCronConfig([mintue, hour, day, e.target.value, week].join(' '))} + onChange={(e) => onChangeCronConfig([minute, hour, day, e.target.value, week].join(' '))} /> onChangeCronConfig([mintue, hour, day, month, e.target.value].join(' '))} + onChange={(e) => onChangeCronConfig([minute, hour, day, month, e.target.value].join(' '))} /> diff --git a/config-ui/src/routes/blueprint/detail/components/sync-policy/styled.ts b/config-ui/src/routes/blueprint/detail/components/sync-policy/styled.ts index 079e5fc8a4a..e10081c31dc 100644 --- a/config-ui/src/routes/blueprint/detail/components/sync-policy/styled.ts +++ b/config-ui/src/routes/blueprint/detail/components/sync-policy/styled.ts @@ -47,5 +47,5 @@ export const Input = styled.div` `; export const Error = styled.div` - color: #e34040; + color: ${({ theme }) => theme.colors.errorAlt}; `; diff --git a/config-ui/src/routes/blueprint/detail/styled.ts b/config-ui/src/routes/blueprint/detail/styled.ts index 6a4ecafb5a4..88dd9a03b30 100644 --- a/config-ui/src/routes/blueprint/detail/styled.ts +++ b/config-ui/src/routes/blueprint/detail/styled.ts @@ -51,7 +51,7 @@ export const ConnectionItem = styled.li` margin-right: 24px; padding: 12px 16px; width: 280px; - background: #ffffff; + background: ${({ theme }) => theme.colors.bgContainer}; box-shadow: 0px 2.4px 4.8px -0.8px rgba(0, 0, 0, 0.1), 0px 1.6px 8px rgba(0, 0, 0, 0.07); border-radius: 4px; diff --git a/config-ui/src/routes/blueprint/home/index.tsx b/config-ui/src/routes/blueprint/home/index.tsx index daa57c1d5b4..0085a435da4 100644 --- a/config-ui/src/routes/blueprint/home/index.tsx +++ b/config-ui/src/routes/blueprint/home/index.tsx @@ -121,7 +121,11 @@ export const BlueprintHomePage = () => { title: 'Blueprint Name', key: 'name', render: (_, { id, name }) => ( - + {name} ), diff --git a/config-ui/src/routes/blueprint/home/styled.ts b/config-ui/src/routes/blueprint/home/styled.ts index 0f4b89498cd..df93ea1811d 100644 --- a/config-ui/src/routes/blueprint/home/styled.ts +++ b/config-ui/src/routes/blueprint/home/styled.ts @@ -35,7 +35,7 @@ export const Label = styled.label` `; export const LabelInfo = styled.i` - color: #ff8b8b; + color: ${({ theme }) => theme.colors.secondary}; `; export const LabelDescription = styled.p` diff --git a/config-ui/src/routes/connection/connection.tsx b/config-ui/src/routes/connection/connection.tsx index d9a5e4b7ab5..7f21a7ff5f8 100644 --- a/config-ui/src/routes/connection/connection.tsx +++ b/config-ui/src/routes/connection/connection.tsx @@ -40,6 +40,7 @@ import { IConnection } from '@/types'; import { operator } from '@/utils'; import * as S from './styled'; +import { getPluginScopeName } from '@/plugins'; const brandName = import.meta.env.DEVLAKE_BRAND_NAME ?? 'DevLake'; @@ -92,7 +93,7 @@ export const Connection = () => { () => [ data?.scopes.map((it: any) => ({ id: getPluginScopeId(plugin, it.scope), - name: it.scope.fullName ?? it.scope.name, + name: getPluginScopeName(plugin, it.scope) || it.scope.fullName || it.scope.name, projects: it.blueprints?.map((bp: any) => bp.projectName) ?? [], configId: it.scopeConfig?.id, configName: it.scopeConfig?.name, @@ -245,10 +246,9 @@ export const Connection = () => { setOperating(false); setVersion((v) => v + 1); setScopeIds([]); - setPage(1) + setPage(1); }; - const handleAssociateScopeConfig = async (trId: ID) => { const [success] = await operator( () => @@ -316,12 +316,14 @@ export const Connection = () => { )} {dataSource.length > 0 && ( - )} @@ -593,19 +595,15 @@ export const Connection = () => { >
-
You are about to delete {scopeIds.length} data scopes:
+
+ You are about to delete {scopeIds.length} data scopes: +
    {scopeIds.slice(0, 5).map((id) => { const scope = dataSource.find((s) => s.id === id); - return ( -
  • - {scope?.name || `Scope ID ${id}`} -
  • - ); + return
  • {scope?.name || `Scope ID ${id}`}
  • ; })} - {scopeIds.length > 5 && ( -
  • ...and {scopeIds.length - 5} more
  • - )} + {scopeIds.length > 5 &&
  • ...and {scopeIds.length - 5} more
  • }
@@ -621,9 +619,7 @@ export const Connection = () => { cancelButtonProps={{ style: { display: 'none' } }} onOk={handleHideDialog} > - +
Progress: {bulkDeleteProgress.completed}/{bulkDeleteProgress.total} diff --git a/config-ui/src/routes/connection/connections.tsx b/config-ui/src/routes/connection/connections.tsx index 776089b0a80..87b358b9ee9 100644 --- a/config-ui/src/routes/connection/connections.tsx +++ b/config-ui/src/routes/connection/connections.tsx @@ -61,8 +61,9 @@ export const Connections = () => { setPlugin(plugin); }; - const handleShowFormDialog = () => { + const handleShowFormDialog = (pluginName?: string) => { setType('form'); + if (pluginName) setPlugin(pluginName); }; const handleHideDialog = () => { @@ -75,7 +76,7 @@ export const Connections = () => { }; return ( - +

Connections

Create and manage data connections from the following data sources or Webhooks to be used in syncing data in @@ -165,10 +166,10 @@ export const Connections = () => { footer={null} onCancel={handleHideDialog} > - + )} - {type === 'form' && pluginConfig && ( + {type === 'form' && plugin && pluginConfig && ( { footer={null} onCancel={handleHideDialog} > - handleSuccessAfter(pluginConfig.plugin, id)} - /> + handleSuccessAfter(plugin, id)} /> )} diff --git a/config-ui/src/routes/connection/styled.ts b/config-ui/src/routes/connection/styled.ts index 6aa45c94808..72e339e5519 100644 --- a/config-ui/src/routes/connection/styled.ts +++ b/config-ui/src/routes/connection/styled.ts @@ -18,7 +18,7 @@ import styled from 'styled-components'; -export const Wrapper = styled.div<{ theme: string }>` +export const Wrapper = styled.div` h2 { margin-top: 36px; } @@ -39,7 +39,7 @@ export const Wrapper = styled.div<{ theme: string }>` left: 0; width: 48px; height: 4px; - background-color: ${({ theme }) => theme}; + background-color: ${({ theme }) => theme.colors.primary}; } } ul { @@ -58,13 +58,15 @@ export const Wrapper = styled.div<{ theme: string }>` padding: 20px 0; width: 160px; border-radius: 8px; + border: 1px solid ${({ theme }) => (theme.mode === 'dark' ? theme.colors.border : 'transparent')}; + background-color: ${({ theme }) => theme.colors.bgContainer}; box-shadow: 0px 2.4px 4.8px -0.8px rgba(0, 0, 0, 0.1), 0px 1.6px 8px rgba(0, 0, 0, 0.07); box-sizing: border-box; cursor: pointer; transition: all 0.2s linear; &:hover { - background-color: #eeeeee; + background-color: ${({ theme }) => theme.colors.bgHover}; } & > .beta { @@ -73,8 +75,8 @@ export const Wrapper = styled.div<{ theme: string }>` right: 0; padding: 4px 8px; font-size: 12px; - color: #fff; - background-color: #f5a623; + color: ${({ theme }) => theme.colors.textInverse}; + background-color: ${({ theme }) => theme.colors.warning}; border-radius: 8px; } @@ -102,12 +104,12 @@ export const Wrapper = styled.div<{ theme: string }>` content: ''; width: 88px; height: 1px; - background-color: #dbdcdf; + background-color: ${({ theme }) => theme.colors.border}; } } & > .count { - color: #70727f; + color: ${({ theme }) => theme.colors.textMuted}; } } `; diff --git a/config-ui/src/routes/db-migrate/index.tsx b/config-ui/src/routes/db-migrate/index.tsx index 7a83b91b535..d32903eacbf 100644 --- a/config-ui/src/routes/db-migrate/index.tsx +++ b/config-ui/src/routes/db-migrate/index.tsx @@ -46,7 +46,7 @@ export const DBMigrate = () => {

- + New Migration Scripts Detected

diff --git a/config-ui/src/routes/error/index.tsx b/config-ui/src/routes/error/index.tsx index 3e1b1dd50c4..9e02bf64ce5 100644 --- a/config-ui/src/routes/error/index.tsx +++ b/config-ui/src/routes/error/index.tsx @@ -33,8 +33,8 @@ export const Error = () => { - -

{error.toString() || 'Unknown Error'}

+ +

{error.toString() || 'Unknown Error'}

Please try again, if the problem persists include the above error message when filing a bug report on{' '} diff --git a/config-ui/src/routes/index.ts b/config-ui/src/routes/index.ts index 0fb0d7e5175..173746252ed 100644 --- a/config-ui/src/routes/index.ts +++ b/config-ui/src/routes/index.ts @@ -22,6 +22,7 @@ export * from './connection'; export * from './db-migrate'; export * from './error'; export * from './layout'; +export * from './login'; export * from './not-found'; export * from './onboard'; export * from './pipeline'; diff --git a/config-ui/src/routes/layout/layout.tsx b/config-ui/src/routes/layout/layout.tsx index adcffc988ed..0d12eb5c3d0 100644 --- a/config-ui/src/routes/layout/layout.tsx +++ b/config-ui/src/routes/layout/layout.tsx @@ -19,15 +19,29 @@ import { useState, useEffect, useMemo } from 'react'; import { useLoaderData, Outlet, useNavigate, useLocation } from 'react-router-dom'; import { Helmet } from 'react-helmet'; -import { Layout as AntdLayout, Menu, Divider } from 'antd'; +import { Layout as AntdLayout, Menu, Divider, Dropdown, Button, Tooltip } from 'antd'; +import { UserOutlined, LogoutOutlined, SunOutlined, MoonOutlined, DesktopOutlined } from '@ant-design/icons'; +import API from '@/api'; import { PageLoading, Logo, ExternalLink } from '@/components'; -import { init, selectError, selectStatus } from '@/features'; +import { init, selectError, selectStatus, cycleMode, selectThemeMode } from '@/features'; import { OnboardCard } from '@/routes/onboard/components'; import { useAppDispatch, useAppSelector } from '@/hooks'; import { menuItems, menuItemsMatch, headerItems } from './config'; +const themeIcon = { + light: , + dark: , + system: , +} as const; + +const themeLabel = { + light: 'Light theme', + dark: 'Dark theme', + system: 'Follow system', +} as const; + const { Sider, Header, Content, Footer } = AntdLayout; const brandName = import.meta.env.DEVLAKE_BRAND_NAME ?? 'DevLake'; @@ -36,7 +50,24 @@ export const Layout = () => { const [openKeys, setOpenKeys] = useState([]); const [selectedKeys, setSelectedKeys] = useState([]); - const { version, plugins } = useLoaderData() as { version: string; plugins: string[] }; + const { version, plugins, user } = useLoaderData() as { + version: string; + plugins: string[]; + user: { authenticated: boolean; name: string; email: string } | null; + }; + + const handleLogout = async () => { + try { + const res = await API.auth.logout(); + if (res.logoutUrl) { + window.location.href = res.logoutUrl; + return; + } + } catch (e) { + // fall through to /login regardless + } + window.location.href = '/login'; + }; const navigate = useNavigate(); const { pathname } = useLocation(); @@ -44,6 +75,7 @@ export const Layout = () => { const dispatch = useAppDispatch(); const status = useAppSelector(selectStatus); const error = useAppSelector(selectError); + const themeMode = useAppSelector(selectThemeMode); useEffect(() => { dispatch(init(plugins)); @@ -134,6 +166,37 @@ export const Layout = () => { {i !== arr.length - 1 && } ))} + + + + + + )}

diff --git a/config-ui/src/routes/layout/loader.ts b/config-ui/src/routes/layout/loader.ts index b52021b41c1..2aa81b23351 100644 --- a/config-ui/src/routes/layout/loader.ts +++ b/config-ui/src/routes/layout/loader.ts @@ -34,14 +34,16 @@ export const layoutLoader = async ({ request }: Props) => { } let fePlugins = getRegisterPlugins(); - const bePlugins = await API.plugin.list(); - try { const envPlugins = import.meta.env.DEVLAKE_PLUGINS.split(',').filter(Boolean); fePlugins = fePlugins.filter((plugin) => !envPlugins.length || envPlugins.includes(plugin)); } catch (err) {} - const res = await API.version(request.signal); + const [bePlugins, res, user] = await Promise.all([ + API.plugin.list(), + API.version(request.signal), + API.auth.userinfo().catch(() => null), + ]); return { version: res.version, @@ -49,5 +51,6 @@ export const layoutLoader = async ({ request }: Props) => { fePlugins, bePlugins.map((it) => it.plugin), ), + user, }; }; diff --git a/config-ui/src/routes/login/index.tsx b/config-ui/src/routes/login/index.tsx new file mode 100644 index 00000000000..cfa9aa2ae91 --- /dev/null +++ b/config-ui/src/routes/login/index.tsx @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { useEffect, useState } from 'react'; +import { Card, Button, Typography, Alert, Space } from 'antd'; + +import API from '@/api'; +import type { Methods, Provider } from '@/api/auth'; +import { TipLayout } from '@/components'; +import { DEVLAKE_ENDPOINT } from '@/config'; + +const { Title, Paragraph } = Typography; + +export const Login = () => { + const [methods, setMethods] = useState(null); + const [error, setError] = useState(null); + + const params = new URLSearchParams(window.location.search); + const returnUrl = params.get('return_url') || '/'; + + useEffect(() => { + API.auth + .methods() + .then(setMethods) + .catch((e) => setError(e?.message ?? 'Failed to load login methods')); + }, []); + + const startOIDC = (p: Provider) => { + const sep = p.loginUrl.includes('?') ? '&' : '?'; + window.location.href = `${DEVLAKE_ENDPOINT}${p.loginUrl}${sep}return_url=${encodeURIComponent(returnUrl)}`; + }; + + const providers = methods?.providers ?? []; + const apiKey = methods?.apiKey; + const noProviders = providers.length === 0 && !apiKey?.enabled; + + return ( + + + + Sign in to DevLake + + {error && } + {providers.length > 0 && ( + + {providers.map((p) => ( + + ))} + + )} + {providers.length === 0 && apiKey?.enabled && ( + + Single Sign-On is not configured. Use an API key (Authorization: Bearer ...) to access /rest endpoints, or + ask your administrator to enable OIDC. + + )} + {noProviders && methods !== null && ( + + )} + + + ); +}; diff --git a/config-ui/src/routes/not-found/index.tsx b/config-ui/src/routes/not-found/index.tsx index e9645c3a9f2..3f1726c5970 100644 --- a/config-ui/src/routes/not-found/index.tsx +++ b/config-ui/src/routes/not-found/index.tsx @@ -30,7 +30,7 @@ export const NotFound = () => {

- + 404 Not Found

diff --git a/config-ui/src/routes/onboard/components/card.tsx b/config-ui/src/routes/onboard/components/card.tsx index 3c59ee082d2..2d45ffe3eab 100644 --- a/config-ui/src/routes/onboard/components/card.tsx +++ b/config-ui/src/routes/onboard/components/card.tsx @@ -32,7 +32,7 @@ interface Props { } export const OnboardCard = ({ style }: Props) => { - const [oeprating, setOperating] = useState(false); + const [operating, setOperating] = useState(false); const [version, setVersion] = useState(0); const navigate = useNavigate(); @@ -49,11 +49,11 @@ export const OnboardCard = ({ style }: Props) => { const tasksRes = useAutoRefresh( async () => { - if ((data && data.done) || !record) { + if ((data && data.done) || !record || !record.pipelineId) { return; } - return await API.pipeline.subTasks(record?.pipelineId as string); + return await API.pipeline.subTasks(record.pipelineId as string); }, [record], { @@ -91,7 +91,7 @@ export const OnboardCard = ({ style }: Props) => { title: 'Permanently close this entry?', content: 'You will not be able to get back to the onboarding session again.', okButtonProps: { - loading: oeprating, + loading: operating, }, okText: 'Confirm', onOk: async () => { diff --git a/config-ui/src/routes/onboard/index.tsx b/config-ui/src/routes/onboard/index.tsx index 061be20331f..1d8b1a8bc3c 100644 --- a/config-ui/src/routes/onboard/index.tsx +++ b/config-ui/src/routes/onboard/index.tsx @@ -123,13 +123,13 @@ export const Onboard = ({ logo, title }: Props) => { <>

Connect to your first repository

- +
{[1, 2, 3].includes(step) && ( {steps.map((it) => ( - + {it.step} {it.title} diff --git a/config-ui/src/routes/onboard/step-0.tsx b/config-ui/src/routes/onboard/step-0.tsx index 5e504bd42db..6b88cc9fb88 100644 --- a/config-ui/src/routes/onboard/step-0.tsx +++ b/config-ui/src/routes/onboard/step-0.tsx @@ -115,7 +115,7 @@ export const Step0 = ({ logo = , title = 'DevLake {contextHolder}
{logo} - +

diff --git a/config-ui/src/routes/onboard/step-3.tsx b/config-ui/src/routes/onboard/step-3.tsx index ae010b0f7a7..7218291826d 100644 --- a/config-ui/src/routes/onboard/step-3.tsx +++ b/config-ui/src/routes/onboard/step-3.tsx @@ -22,7 +22,7 @@ import dayjs from 'dayjs'; import API from '@/api'; import { Markdown } from '@/components'; -import { DataScopeRemote, getPluginScopeId } from '@/plugins'; +import { DataScopeRemote, getPluginScopeId, getPluginScopeName } from '@/plugins'; import { operator, formatTime } from '@/utils'; import { Context } from './context'; @@ -121,7 +121,7 @@ export const Step3 = () => { ...it, blueprintId: blueprint.id, pipelineId: pipeline.pipelines[0].id, - scopeName: scopes[0]?.fullName ?? scopes[0].name, + scopeName: getPluginScopeName(plugin, scopes[0]) || scopes[0]?.fullName || scopes[0]?.name, }, ); diff --git a/config-ui/src/routes/onboard/step-4.tsx b/config-ui/src/routes/onboard/step-4.tsx index eb1be96880c..eb6eb44ebd4 100644 --- a/config-ui/src/routes/onboard/step-4.tsx +++ b/config-ui/src/routes/onboard/step-4.tsx @@ -80,6 +80,7 @@ export const DashboardURLMap: Record = { gitlab: import.meta.env.DEVLAKE_DASHBOARD_URL_GITLAB, bitbucket: import.meta.env.DEVLAKE_DASHBOARD_URL_BITBUCKET, azuredevops: import.meta.env.DEVLAKE_DASHBOARD_URL_AZUREDEVOPS, + asana: import.meta.env.DEVLAKE_DASHBOARD_URL_ASANA, }; const getStatus = (data: any) => { @@ -111,7 +112,10 @@ export const Step4 = () => { const { data } = useAutoRefresh( async () => { - return await API.pipeline.subTasks(record?.pipelineId as string); + if (!record?.pipelineId) { + return; + } + return await API.pipeline.subTasks(record.pipelineId as string); }, [record], { diff --git a/config-ui/src/routes/onboard/styled.ts b/config-ui/src/routes/onboard/styled.ts index b8d95a9e798..1ff0222b283 100644 --- a/config-ui/src/routes/onboard/styled.ts +++ b/config-ui/src/routes/onboard/styled.ts @@ -21,7 +21,7 @@ import styled from 'styled-components'; export const Wrapper = styled.div` width: 100%; height: 100vh; - background-color: #f9f9fa; + background-color: ${({ theme }) => theme.colors.bgLayout}; `; export const Inner = styled.div` @@ -49,7 +49,7 @@ export const Step = styled.ul` margin-bottom: 50px; `; -export const StepItem = styled.li<{ $actived: boolean; $activedColor: string }>` +export const StepItem = styled.li<{ $activated: boolean; $activatedColor: string }>` display: flex; align-items: center; position: relative; @@ -61,23 +61,23 @@ export const StepItem = styled.li<{ $actived: boolean; $activedColor: string }>` margin-right: 8px; width: 32px; height: 32px; - color: rgba(0, 0, 0, 0.25); - border: 1px solid rgba(0, 0, 0, 0.25); + color: ${({ theme }) => theme.colors.textDisabled}; + border: 1px solid ${({ theme }) => theme.colors.borderStep}; border-radius: 50%; - ${({ $actived, $activedColor }) => - $actived + ${({ $activated, $activatedColor, theme }) => + $activated ? ` - color: #fff; - background-color: ${$activedColor}; + color: ${theme.colors.textInverse}; + background-color: ${$activatedColor}; border: none; ` : ''} } span:last-child { - ${({ $actived }) => - $actived + ${({ $activated }) => + $activated ? ` font-size: 24px; font-weight: 600;` @@ -91,7 +91,7 @@ export const StepItem = styled.li<{ $actived: boolean; $activedColor: string }>` left: -150px; width: 100px; height: 1px; - background-color: rgba(0, 0, 0, 0.25); + background-color: ${({ theme }) => theme.colors.borderStep}; } &:first-child::before { @@ -102,7 +102,7 @@ export const StepItem = styled.li<{ $actived: boolean; $activedColor: string }>` export const StepContent = styled.div` display: flex; height: 450px; - background-color: #fff; + background-color: ${({ theme }) => theme.colors.bgContainer}; box-shadow: 0px 2.4px 4.8px -0.8px rgba(0, 0, 0, 0.1), 0px 1.6px 8px 0px rgba(0, 0, 0, 0.07); .content { @@ -115,7 +115,7 @@ export const StepContent = styled.div` margin: 12px 0; padding: 0 24px; font-size: 14px; - border-left: 1px solid #f0f0f0; + border-left: 1px solid ${({ theme }) => theme.colors.borderSubtle}; overflow-y: auto; img { @@ -142,7 +142,7 @@ export const StepContent = styled.div` } p { - color: #6c6c6c; + color: ${({ theme }) => theme.colors.textBody}; } code { @@ -151,8 +151,8 @@ export const StepContent = styled.div` font-family: Menlo; line-height: 20px; border-radius: 3px; - border: #f0f0f0; - background: #f5f5f5; + border: 1px solid ${({ theme }) => theme.colors.borderSubtle}; + background: ${({ theme }) => theme.colors.bgCode}; } } `; diff --git a/config-ui/src/routes/pipeline/styled.ts b/config-ui/src/routes/pipeline/styled.ts index 5af0c40e801..95873a8ee60 100644 --- a/config-ui/src/routes/pipeline/styled.ts +++ b/config-ui/src/routes/pipeline/styled.ts @@ -21,19 +21,19 @@ import styled from 'styled-components'; export const StatusWrapper = styled.div` &.ready, &.cancel { - color: #94959f; + color: ${({ theme }) => theme.colors.textTertiary}; } &.loading { - color: #7497f7; + color: ${({ theme }) => theme.colors.primary}; } &.success { - color: #4db764; + color: ${({ theme }) => theme.colors.success}; } &.error { - color: #f5222d; + color: ${({ theme }) => theme.colors.error}; } `; @@ -54,7 +54,7 @@ export const Info = styled.div` & > span { font-size: 12px; - color: #94959f; + color: ${({ theme }) => theme.colors.textTertiary}; text-align: center; } @@ -68,7 +68,7 @@ export const Info = styled.div` p.message { margin: 8px 0 0; - color: #f5222d; + color: ${({ theme }) => theme.colors.error}; } `; @@ -100,23 +100,23 @@ export const TasksHeader = styled.ul` &.ready, &.cancel { - color: #94959f; - background-color: #f9f9fa; + color: ${({ theme }) => theme.colors.textTertiary}; + background-color: ${({ theme }) => theme.colors.bgStatus}; } &.loading { - color: #7497f7; - background-color: #e9efff; + color: ${({ theme }) => theme.colors.primary}; + background-color: ${({ theme }) => theme.colors.infoBg}; } &.success { - color: #4db764; - background-color: #edfbf0; + color: ${({ theme }) => theme.colors.success}; + background-color: ${({ theme }) => theme.colors.successBg}; } &.error { - color: #e34040; - background-color: #feefef; + color: ${({ theme }) => theme.colors.errorAlt}; + background-color: ${({ theme }) => theme.colors.errorBg}; } } @@ -146,7 +146,7 @@ export const Task = styled.div` justify-content: space-between; padding: 16px 0; height: 80px; - border-bottom: 1px solid #dbe4fd; + border-bottom: 1px solid ${({ theme }) => theme.colors.borderTask}; box-sizing: border-box; .info { @@ -181,7 +181,7 @@ export const Task = styled.div` text-overflow: ellipsis; &.error { - color: #f5222d; + color: ${({ theme }) => theme.colors.error}; } } } diff --git a/config-ui/src/routes/project/home/index.tsx b/config-ui/src/routes/project/home/index.tsx index 205ca675c7b..c9c13c839d0 100644 --- a/config-ui/src/routes/project/home/index.tsx +++ b/config-ui/src/routes/project/home/index.tsx @@ -170,7 +170,7 @@ export const ProjectHomePage = () => { {name} diff --git a/config-ui/src/theme/ThemeProvider.tsx b/config-ui/src/theme/ThemeProvider.tsx new file mode 100644 index 00000000000..cd825b3d538 --- /dev/null +++ b/config-ui/src/theme/ThemeProvider.tsx @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { ReactNode, useEffect, useMemo, useState } from 'react'; +import { ConfigProvider } from 'antd'; +import { ThemeProvider as StyledThemeProvider } from 'styled-components'; + +import { useAppSelector } from '@/hooks'; +import { selectThemeMode, resolveThemeMode } from '@/features/theme/slice'; + +import { getTheme, ResolvedTheme } from './tokens'; + +interface Props { + children: ReactNode; +} + +const CSS_VAR_MAP: Record['colors']> = { + '--devlake-color-text': 'text', + '--devlake-color-text-muted': 'textTertiary', + '--devlake-color-text-body': 'textBody', + '--devlake-color-text-subdued': 'textMuted', + '--devlake-color-bg': 'bgContainer', + '--devlake-color-bg-elevated': 'bgElevated', + '--devlake-color-border': 'border', + '--devlake-color-success': 'success', + '--devlake-color-error': 'error', + '--devlake-color-error-alt': 'errorAlt', + '--devlake-color-warning': 'warning', + '--devlake-color-warning-alt': 'warningAlt', + '--devlake-color-warning-soft': 'warningSoft', +}; + +export const ThemeProvider = ({ children }: Props) => { + const mode = useAppSelector(selectThemeMode); + const [resolved, setResolved] = useState(() => resolveThemeMode(mode)); + + useEffect(() => { + setResolved(resolveThemeMode(mode)); + + if (mode !== 'system' || typeof window === 'undefined' || typeof window.matchMedia !== 'function') { + return; + } + + const media = window.matchMedia('(prefers-color-scheme: dark)'); + const onChange = () => setResolved(media.matches ? 'dark' : 'light'); + media.addEventListener('change', onChange); + return () => media.removeEventListener('change', onChange); + }, [mode]); + + const customPrimary = import.meta.env.DEVLAKE_COLOR_CUSTOM; + const theme = useMemo(() => getTheme(resolved, customPrimary), [resolved, customPrimary]); + + useEffect(() => { + if (typeof document === 'undefined') return; + const root = document.documentElement; + root.dataset.theme = resolved; + root.style.colorScheme = resolved; + for (const [cssVar, key] of Object.entries(CSS_VAR_MAP)) { + root.style.setProperty(cssVar, theme.colors[key]); + } + }, [resolved, theme]); + + return ( + + {children} + + ); +}; diff --git a/config-ui/src/theme/index.ts b/config-ui/src/theme/index.ts new file mode 100644 index 00000000000..0ae87ef543c --- /dev/null +++ b/config-ui/src/theme/index.ts @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './tokens'; +export * from './ThemeProvider'; diff --git a/config-ui/src/theme/styled.d.ts b/config-ui/src/theme/styled.d.ts new file mode 100644 index 00000000000..60a567f77d2 --- /dev/null +++ b/config-ui/src/theme/styled.d.ts @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import 'styled-components'; +import type { AppThemeColors, ResolvedTheme } from './tokens'; + +declare module 'styled-components' { + export interface DefaultTheme { + mode: ResolvedTheme; + colors: AppThemeColors; + } +} diff --git a/config-ui/src/theme/tokens.ts b/config-ui/src/theme/tokens.ts new file mode 100644 index 00000000000..903ec2df5d9 --- /dev/null +++ b/config-ui/src/theme/tokens.ts @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { theme as antdTheme, ThemeConfig } from 'antd'; + +export type ThemeMode = 'light' | 'dark' | 'system'; +export type ResolvedTheme = 'light' | 'dark'; + +const primaryShades = { + lightest: '#DBE4FD', + lighter: '#BDCEFB', + light: '#99B3F9', + base: '#7497F7', + dark: '#5D7CD2', + darker: '#4C66AD', + darkest: '#3C5088', + tint100: '#F0F4FE', +}; + +export type AppThemeColors = { + // Brand + primary: string; + primaryHover: string; + + // Status / accent + success: string; + successBg: string; + error: string; + errorAlt: string; + errorMuted: string; + errorBg: string; + warning: string; + warningAlt: string; + warningSoft: string; + secondary: string; + infoBg: string; + + // Surfaces + bgLayout: string; + bgContainer: string; + bgElevated: string; + bgHover: string; + bgMuted: string; + bgCode: string; + bgStatus: string; + + // Borders + border: string; + borderSubtle: string; + borderTask: string; + borderStep: string; + + // Text + text: string; + textSecondary: string; + textTertiary: string; + textMuted: string; + textBody: string; + textFaint: string; + textInverse: string; + textDisabled: string; +}; + +export interface AppTheme { + mode: ResolvedTheme; + colors: AppThemeColors; + antd: ThemeConfig; +} + +const lightColors: AppThemeColors = { + primary: primaryShades.base, + primaryHover: primaryShades.dark, + + success: '#4DB764', + successBg: '#EDFBF0', + error: '#F5222D', + errorAlt: '#E34040', + errorMuted: '#CD4246', + errorBg: '#FEEFEF', + warning: '#F5A623', + warningAlt: '#FAAD14', + warningSoft: '#F4BE55', + secondary: '#FF8B8B', + infoBg: '#E9EFFF', + + bgLayout: '#F9F9FA', + bgContainer: '#FFFFFF', + bgElevated: '#F3F3F3', + bgHover: '#EEEEEE', + bgMuted: '#EFEFEF', + bgCode: '#F5F5F5', + bgStatus: '#F9F9FA', + + border: '#DBDCDF', + borderSubtle: '#F0F0F0', + borderTask: '#DBE4FD', + borderStep: 'rgba(0, 0, 0, 0.25)', + + text: '#292B3F', + textSecondary: '#4D4E5F', + textTertiary: '#94959F', + textMuted: '#70727F', + textBody: '#6C6C6C', + textFaint: '#AAAAAA', + textInverse: '#FFFFFF', + textDisabled: 'rgba(0, 0, 0, 0.25)', +}; + +const darkColors: AppThemeColors = { + primary: primaryShades.base, + primaryHover: primaryShades.lighter, + + success: '#4DB764', + successBg: 'rgba(77, 183, 100, 0.15)', + error: '#F5222D', + errorAlt: '#FF6B6B', + errorMuted: '#FF8080', + errorBg: 'rgba(245, 34, 45, 0.15)', + warning: '#F5A623', + warningAlt: '#FAAD14', + warningSoft: '#F4BE55', + secondary: '#FF8B8B', + infoBg: primaryShades.darkest, + + bgLayout: '#1B1B1D', + bgContainer: '#242526', + bgElevated: '#2D2D2F', + bgHover: '#303032', + bgMuted: '#2D2D2F', + bgCode: '#2D2D2F', + bgStatus: '#2D2D2F', + + border: '#3A3A3C', + borderSubtle: '#2D2D2F', + borderTask: primaryShades.darkest, + borderStep: '#3A3A3C', + + text: '#E3E3E6', + textSecondary: '#BDCEFB', + textTertiary: '#94959F', + textMuted: '#A0A1A8', + textBody: '#A0A1A8', + textFaint: '#70727F', + textInverse: '#1B1B1D', + textDisabled: '#70727F', +}; + +const buildAntdConfig = (colors: AppThemeColors, mode: ResolvedTheme): ThemeConfig => ({ + algorithm: mode === 'dark' ? antdTheme.darkAlgorithm : antdTheme.defaultAlgorithm, + token: { + colorPrimary: colors.primary, + colorSuccess: colors.success, + colorError: colors.error, + colorWarning: colors.warning, + colorBgLayout: colors.bgLayout, + colorBgContainer: colors.bgContainer, + colorBgElevated: colors.bgElevated, + colorBorder: colors.border, + colorBorderSecondary: colors.borderSubtle, + colorText: colors.text, + colorTextSecondary: colors.textSecondary, + colorTextTertiary: colors.textTertiary, + colorTextDisabled: colors.textDisabled, + }, +}); + +const buildTheme = (mode: ResolvedTheme, customPrimary?: string): AppTheme => { + const base = mode === 'dark' ? darkColors : lightColors; + const colors = customPrimary ? { ...base, primary: customPrimary, primaryHover: customPrimary } : base; + return { mode, colors, antd: buildAntdConfig(colors, mode) }; +}; + +export const getTheme = (mode: ResolvedTheme, customPrimary?: string): AppTheme => buildTheme(mode, customPrimary); diff --git a/config-ui/src/types/connection.ts b/config-ui/src/types/connection.ts index 04551002ea8..17a3b85af0e 100644 --- a/config-ui/src/types/connection.ts +++ b/config-ui/src/types/connection.ts @@ -16,12 +16,18 @@ * */ +export interface ICustomHeader { + key: string; + value: string; +} + export interface IConnectionAPI { id: ID; name: string; endpoint: string; authMethod?: string; token?: string; + adminApiKey?: string; username?: string; password?: string; appId?: string; @@ -31,6 +37,8 @@ export interface IConnectionAPI { proxy: string; rateLimitPerHour?: number; organization?: string; + organizationId?: string; + customHeaders?: ICustomHeader[]; } export interface IConnectionTestResult { @@ -76,6 +84,7 @@ export interface IConnection { endpoint: string; authMethod?: string; token?: string; + adminApiKey?: string; username?: string; password?: string; appId?: string; @@ -85,4 +94,6 @@ export interface IConnection { proxy: string; rateLimitPerHour?: number; organization?: string; + organizationId?: string; + customHeaders?: ICustomHeader[]; } diff --git a/config-ui/src/utils/request.ts b/config-ui/src/utils/request.ts index a3d2e5275ef..e91739f7242 100644 --- a/config-ui/src/utils/request.ts +++ b/config-ui/src/utils/request.ts @@ -23,6 +23,11 @@ import { DEVLAKE_ENDPOINT } from '@/config'; const instance = axios.create({ baseURL: DEVLAKE_ENDPOINT, + withCredentials: true, + // Double-submit CSRF: axios reads `devlake_csrf` and echoes it as + // `X-CSRF-Token` on unsafe methods. The cookie is set by /auth/callback. + xsrfCookieName: 'devlake_csrf', + xsrfHeaderName: 'X-CSRF-Token', }); export type RequestConfig = { @@ -34,6 +39,10 @@ export type RequestConfig = { headers?: Record; }; +const isLoginRoute = () => window.location.pathname.replace(/\/+$/, '').endsWith('/login'); + +let redirectingToLogin = false; + instance.interceptors.response.use( (response) => response, (error) => { @@ -43,6 +52,12 @@ instance.interceptors.response.use( window.location.replace('/db-migrate'); } + if (status === 401 && !isLoginRoute() && !redirectingToLogin) { + redirectingToLogin = true; + const returnUrl = encodeURIComponent(window.location.pathname + window.location.search); + window.location.replace(`/login?return_url=${returnUrl}`); + } + return Promise.reject(error); }, ); diff --git a/config-ui/src/vite-env.d.ts b/config-ui/src/vite-env.d.ts index ceea9065145..946fc8444fd 100644 --- a/config-ui/src/vite-env.d.ts +++ b/config-ui/src/vite-env.d.ts @@ -30,6 +30,7 @@ interface ImportMetaEnv { readonly DEVLAKE_DASHBOARD_URL_GITLAB: string; readonly DEVLAKE_DASHBOARD_URL_BITBUCKET: string; readonly DEVLAKE_DASHBOARD_URL_AZUREDEVOPS: string; + readonly DEVLAKE_DASHBOARD_URL_ASANA: string; } interface ImportMeta { diff --git a/config-ui/vite.config.ts b/config-ui/vite.config.ts index b9def37db0c..5bf4760af99 100644 --- a/config-ui/vite.config.ts +++ b/config-ui/vite.config.ts @@ -21,6 +21,10 @@ import { defineConfig } from 'vite'; import react from '@vitejs/plugin-react'; import svgr from 'vite-plugin-svgr'; +// Allow Grafana access from the dev server when using Dev Container +const grafanaOrigin = process.env.VITE_GRAFANA_URL || 'http://localhost:3002'; +const grafanaChangeOrigin = envBool('VITE_GRAFANA_CHANGE_ORIGIN', true); + // https://vitejs.dev/config/ export default defineConfig({ plugins: [react(), svgr()], @@ -36,8 +40,9 @@ export default defineConfig({ rewrite: (path) => path.replace(/^\/api\//, ''), }, '/grafana': { - target: 'http://localhost:3002/', - changeOrigin: true, + target: grafanaOrigin, + changeOrigin: grafanaChangeOrigin, + ws: true, // Proxying websockets to allow features like query auto-complete }, }, }, @@ -48,3 +53,9 @@ export default defineConfig({ }, }, }); + +function envBool(name: string, defaultValue = false): boolean { + const v = process.env[name]; + if (v == null) return defaultValue; + return /^(1|true|yes|on)$/i.test(v); +} diff --git a/devops/deployment/k8s/k8s-deploy.yaml b/devops/deployment/k8s/k8s-deploy.yaml index b13b97b4695..cea2a1b5f58 100644 --- a/devops/deployment/k8s/k8s-deploy.yaml +++ b/devops/deployment/k8s/k8s-deploy.yaml @@ -32,7 +32,9 @@ data: MYSQL_ROOT_PASSWORD: admin MYSQL_URL: mysql:3306 # grafana deployment - GF_SERVER_ROOT_URL: http://YOURDOMAIN:30002/grafana + GF_SERVER_PROTOCOL: "http" + GF_SERVER_DOMAIN: "YOURDOMAIN:30002" + GF_SERVER_ROOT_URL: "http://YOURDOMAIN:30002/grafana" GF_USERS_DEFAULT_THEME: "light" # for devlake deployment DB_URL: mysql://merico:merico@mysql:3306/lake?charset=utf8mb4&parseTime=True diff --git a/devops/docker/lake-builder/Dockerfile b/devops/docker/lake-builder/Dockerfile index c09a0088630..be6e31f3d7e 100644 --- a/devops/docker/lake-builder/Dockerfile +++ b/devops/docker/lake-builder/Dockerfile @@ -18,7 +18,7 @@ FROM --platform=linux/amd64 debian:bullseye as debian-amd64 RUN apt-get -y update && apt -y upgrade &&\ apt-get install -y libssh2-1-dev libssl-dev zlib1g-dev -FROM golang:1.20.4-bullseye as builder +FROM golang:1.26-bullseye as builder # Base dependencies RUN apt-get -y update && apt -y upgrade &&\ @@ -55,14 +55,14 @@ COPY --from=builder /tmp/deps/include/ /usr/include/ ENV PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig # Install Golang -RUN curl -L https://git.io/vQhTU | bash -s -- --version 1.23.0 +RUN curl -L https://git.io/vQhTU | bash -s -- --version 1.26.2 RUN mv /root/go /go &&\ mv /root/.go /usr/local/go &&\ ln -sf /usr/local/go/bin/* /usr/bin # Install Golang Tools RUN export GOPATH=/go && \ - go install github.com/vektra/mockery/v2@v2.53.5 && \ + go install github.com/vektra/mockery/v2@v2.53.6 && \ go install github.com/swaggo/swag/cmd/swag@v1.16.1 # Golang Env diff --git a/docker-compose-dev-mysql.yml b/docker-compose-dev-mysql.yml new file mode 100644 index 00000000000..11b9bab0b5f --- /dev/null +++ b/docker-compose-dev-mysql.yml @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: devlake-mysql + +services: + mysql: + image: mysql:8 + volumes: + - mysql-storage:/var/lib/mysql + restart: always + ports: + - 3306:3306 + environment: + MYSQL_ROOT_PASSWORD: admin + MYSQL_DATABASE: lake + MYSQL_USER: merico + MYSQL_PASSWORD: merico + TZ: UTC + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin + --skip-log-bin + + grafana: + image: devlake.docker.scarf.sh/apache/devlake-dashboard:latest + build: + context: grafana/ + ports: + - 3001:3000 + volumes: + - grafana-storage:/var/lib/grafana + - ./grafana/dashboards/mysql:/etc/grafana/dashboards:ro + environment: + GF_SERVER_ROOT_URL: "http://localhost:4001/grafana" + GF_USERS_DEFAULT_THEME: "light" + DATABASE_TYPE: mysql + DATABASE_HOST: mysql + DATABASE_PORT: 3306 + DATABASE_NAME: lake + DATABASE_USER: merico + DATABASE_PASSWORD: merico + TZ: UTC + restart: always + depends_on: + - mysql + + devlake: + image: devlake-local:latest + build: + context: backend + dockerfile: Dockerfile.local + args: + HTTPS_PROXY: "${HTTPS_PROXY}" + GOPROXY: "${GOPROXY}" + ports: + - 8083:8080 + restart: always + volumes: + - devlake-log:/app/logs + env_file: + - ./.env + environment: + DB_URL: mysql://merico:merico@mysql:3306/lake?charset=utf8mb4&parseTime=True&loc=UTC + E2E_DB_URL: mysql://merico:merico@mysql:3306/lake_test?charset=utf8mb4&parseTime=True&loc=UTC + REMOTE_PLUGIN_DIR: "" + FORCE_MIGRATION: "true" + LOGGING_DIR: /app/logs + TZ: UTC + depends_on: + - mysql + + config-ui: + image: devlake.docker.scarf.sh/apache/devlake-config-ui:latest + build: + context: "config-ui" + ports: + - 4001:4000 + env_file: + - ./.env + environment: + DEVLAKE_ENDPOINT: devlake:8080 + GRAFANA_ENDPOINT: grafana:3000 + TZ: UTC + depends_on: + - devlake + + authproxy: + image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3-amd64 + network_mode: "host" + ports: + - 4181:4180 + env_file: + - ./.env + +volumes: + mysql-storage: + grafana-storage: + devlake-log: diff --git a/docker-compose-dev.yml b/docker-compose-dev-postgresql.yml similarity index 58% rename from docker-compose-dev.yml rename to docker-compose-dev-postgresql.yml index b575a6029ac..36b0a34e6e7 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev-postgresql.yml @@ -13,42 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -version: "3" -services: - mysql: - image: mysql:8 - volumes: - - mysql-storage:/var/lib/mysql - restart: always - ports: - - 3306:3306 - environment: - MYSQL_ROOT_PASSWORD: admin - MYSQL_DATABASE: lake - MYSQL_USER: merico - MYSQL_PASSWORD: merico - TZ: UTC - command: --character-set-server=utf8mb4 - --collation-server=utf8mb4_bin - --skip-log-bin +name: devlake-postgresql - postgres: - image: postgres:14.2 - volumes: - - postgres-storage:/var/lib/postgresql - restart: always - ports: - - 5432:5432 - environment: - POSTGRES_DB: lake - POSTGRES_USER: merico - POSTGRES_PASSWORD: merico - TZ: UTC - - postgres17: +services: + postgres: image: postgres:17.2 volumes: - - postgres17-storage:/var/lib/postgresql + - postgres-storage:/var/lib/postgresql restart: always ports: - 5432:5432 @@ -66,17 +37,20 @@ services: - 3002:3000 volumes: - grafana-storage:/var/lib/grafana + - ./grafana/dashboards/postgresql:/etc/grafana/dashboards:ro environment: GF_SERVER_ROOT_URL: "http://localhost:4000/grafana" GF_USERS_DEFAULT_THEME: "light" - MYSQL_URL: mysql:3306 - MYSQL_DATABASE: lake - MYSQL_USER: merico - MYSQL_PASSWORD: merico + DATABASE_TYPE: postgresql + DATABASE_HOST: postgres + DATABASE_PORT: 5432 + DATABASE_NAME: lake + DATABASE_USER: merico + DATABASE_PASSWORD: merico TZ: UTC restart: always depends_on: - - mysql + - postgres devlake: image: devlake-local:latest @@ -87,7 +61,7 @@ services: HTTPS_PROXY: "${HTTPS_PROXY}" GOPROXY: "${GOPROXY}" ports: - - 8080:8080 + - 8084:8080 restart: always volumes: - devlake-log:/app/logs @@ -100,9 +74,8 @@ services: FORCE_MIGRATION: "true" LOGGING_DIR: /app/logs TZ: UTC - # LOGOUT_URI: https://xxx.amazoncognito.com/logout?client_id=yyy&logout_uri=http%3A%2F%2Flocalhost%3A4180%2Foauth2%2Fsign_out depends_on: - - mysql + - postgres config-ui: image: devlake.docker.scarf.sh/apache/devlake-config-ui:latest @@ -116,37 +89,18 @@ services: DEVLAKE_ENDPOINT: devlake:8080 GRAFANA_ENDPOINT: grafana:3000 TZ: UTC - #ADMIN_USER: devlake - #ADMIN_PASS: letsdivein depends_on: - devlake authproxy: - image: quay.io/oauth2-proxy/oauth2-proxy:v7.4.0-amd64 + image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3-amd64 network_mode: "host" ports: - 4180:4180 env_file: - ./.env - # environment: - # OAUTH2_PROXY_PROVIDER: oidc - # OAUTH2_PROXY_PROVIDER_DISPLAY_NAME: my provider - # OAUTH2_PROXY_COOKIE_SECRET: - # OAUTH2_PROXY_COOKIE_DOMAINS: localhost:4180 - # OAUTH2_PROXY_COOKIE_SECURE: 'false' - # OAUTH2_PROXY_EMAIL_DOMAINS: * - # OAUTH2_PROXY_OIDC_ISSUER_URL: - # OAUTH2_PROXY_OIDC_JWKS_URL: - # OAUTH2_PROXY_CLIENT_ID: - # OAUTH2_PROXY_CLIENT_SECRET: - # OAUTH2_PROXY_UPSTREAMS: http://localhost:4000 - # OAUTH2_PROXY_HTTP_ADDRESS: http://0.0.0.0:4180 - # OAUTH2_PROXY_REVERSE_PROXY: 'true' - # OAUTH2_PROXY_SKIP_AUTH_ROUTES: ^/grafana.* volumes: - mysql-storage: - grafana-storage: postgres-storage: - postgres17-storage: - devlake-log: \ No newline at end of file + grafana-storage: + devlake-log: diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 00000000000..af179c7b6e7 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,16 @@ +{ + "name": "e2e", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "dependencies": { + "@playwright/test": "^1.58.2" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 00000000000..d4006283687 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { defineConfig } from '@playwright/test'; + +export default defineConfig({ + testDir: '.', + testMatch: '*.spec.ts', + timeout: 180000, + expect: { + timeout: 10000, + }, + use: { + baseURL: 'http://localhost:4000', + screenshot: 'on', + trace: 'on-first-retry', + }, + reporter: [['html', { open: 'never' }], ['list']], + projects: [ + { + name: 'chromium', + use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } }, + }, + ], +}); diff --git a/e2e/qdev-full-flow.spec.ts b/e2e/qdev-full-flow.spec.ts new file mode 100644 index 00000000000..0498df67baa --- /dev/null +++ b/e2e/qdev-full-flow.spec.ts @@ -0,0 +1,248 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { test, expect, request, Page } from '@playwright/test'; +import * as path from 'path'; +import * as fs from 'fs'; + +const API = 'http://localhost:8080'; +const UI = 'http://localhost:4000'; +const GRAFANA = 'http://localhost:3002'; +const SCREENSHOT_DIR = path.join(__dirname, 'screenshots'); + +// Use existing connection with valid credentials +const EXISTING_CONNECTION_ID = 5; + +const state: { + connectionId: number; + scopeId: string; + blueprintId: number; + pipelineId: number; +} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0, pipelineId: 0 }; + +fs.mkdirSync(SCREENSHOT_DIR, { recursive: true }); + +async function grafanaLogin(page: Page) { + await page.goto(`${GRAFANA}/grafana/login`); + await page.waitForLoadState('networkidle'); + if (page.url().includes('/login')) { + await page.locator('input[name="user"]').fill('admin'); + await page.locator('input[name="password"]').fill('admin'); + await page.locator('button[type="submit"]').click(); + await page.waitForTimeout(2000); + // Handle "change password" prompt if shown + const skipBtn = page.locator('a:has-text("Skip")'); + if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) { + await skipBtn.click(); + } + await page.waitForTimeout(1000); + } +} + +async function openGrafanaDashboard(page: Page, uid: string, screenshotPath: string) { + await grafanaLogin(page); + await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`); + + // Wait for first panel data to load + try { + await page.waitForResponse( + (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200, + { timeout: 30000 } + ); + } catch { /* some dashboards may not fire queries immediately */ } + + // Wait for rendering to settle + await page.waitForTimeout(5000); + + // Take viewport screenshot (top section) + await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') }); + + // Scroll down and take more sections + const scrollHeight = await page.evaluate(() => document.body.scrollHeight); + let section = 1; + for (let y = 900; y < scrollHeight; y += 900) { + await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y); + await page.waitForTimeout(3000); + section++; + await page.screenshot({ path: screenshotPath.replace('.png', `-section${section}.png`) }); + } + + // Also take full page screenshot + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(2000); + await page.screenshot({ path: screenshotPath, fullPage: true }); +} + +test.describe.serial('Q-Dev Plugin Full Flow', () => { + + test('Step 1: Verify Existing Connection via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.get(`/plugins/q_dev/connections/${state.connectionId}`); + expect(resp.ok()).toBeTruthy(); + const conn = await resp.json(); + console.log(`Using connection: id=${conn.id}, name=${conn.name}, bucket=${conn.bucket}`); + + const testResp = await api.post(`/plugins/q_dev/connections/${state.connectionId}/test`); + const testBody = await testResp.json(); + console.log('Test connection:', testBody.success ? 'OK' : testBody.message); + expect(testResp.ok()).toBeTruthy(); + }); + + test('Step 2: View Config-UI Home', async ({ page }) => { + await page.goto(UI); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1000); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '01-config-ui-home.png'), fullPage: true }); + console.log('Screenshot: Config-UI home'); + }); + + test('Step 3: Create Scope (S3 Slice) via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.put(`/plugins/q_dev/connections/${state.connectionId}/scopes`, { + data: { + data: [ + { + accountId: '034362076319', + basePath: '', + year: 2026, + month: 3, + }, + ], + }, + }); + + const body = await resp.json(); + console.log('Scope created:', resp.status()); + expect(resp.ok()).toBeTruthy(); + state.scopeId = body[0]?.id; + expect(state.scopeId).toBeTruthy(); + console.log(`Scope id: ${state.scopeId}`); + }); + + test('Step 4: Create Blueprint via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post('/blueprints', { + data: { + name: `e2e-blueprint-${Date.now()}`, + mode: 'NORMAL', + enable: true, + cronConfig: '0 0 * * *', + isManual: true, + connections: [ + { + pluginName: 'q_dev', + connectionId: state.connectionId, + scopes: [{ scopeId: state.scopeId }], + }, + ], + }, + }); + + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.blueprintId = body.id; + console.log(`Blueprint created: id=${state.blueprintId}`); + }); + + test('Step 5: Trigger Pipeline via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, { data: {} }); + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.pipelineId = body.id; + console.log(`Pipeline triggered: id=${state.pipelineId}`); + }); + + test('Step 6: Wait for Pipeline to Complete', async () => { + const api = await request.newContext({ baseURL: API }); + const maxWait = 120000; + const start = Date.now(); + let status = ''; + + while (Date.now() - start < maxWait) { + const resp = await api.get(`/pipelines/${state.pipelineId}`); + const pipeline = await resp.json(); + status = pipeline.status; + console.log(`Pipeline status: ${status} (${Math.round((Date.now() - start) / 1000)}s)`); + if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + + // Print task details + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + if (tasksResp.ok()) { + const { tasks } = await tasksResp.json(); + for (const t of tasks || []) { + console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed: ${t.failedSubTask})` : ''}`); + if (t.message) console.log(` Error: ${t.message.substring(0, 300)}`); + } + } + + expect(status).toBe('TASK_COMPLETED'); + }); + + test('Step 7: Verify Data via MySQL', async () => { + const api = await request.newContext({ baseURL: API }); + + // Use pipeline tasks to confirm data was processed + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + const { tasks } = await tasksResp.json(); + expect(tasks[0].status).toBe('TASK_COMPLETED'); + console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`); + }); + + test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_user_report', path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png')); + console.log('Screenshot: Kiro Usage Dashboard'); + }); + + test('Step 9: Grafana - Kiro Legacy Feature Metrics', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_feature_metrics', path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png')); + console.log('Screenshot: Kiro Legacy Feature Metrics'); + }); + + test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_logging', path.join(SCREENSHOT_DIR, '04-dashboard-logging.png')); + console.log('Screenshot: Kiro AI Activity Insights'); + }); + + test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => { + await openGrafanaDashboard(page, 'qdev_executive', path.join(SCREENSHOT_DIR, '05-dashboard-executive.png')); + console.log('Screenshot: Kiro Executive Dashboard'); + }); + + test('Step 12: View Pipeline in Config-UI', async ({ page }) => { + // Navigate to the API proxy route for pipelines + await page.goto(`${UI}/api/pipelines?pageSize=5`); + await page.waitForLoadState('networkidle'); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '06-config-ui-pipelines.png'), fullPage: true }); + console.log('Screenshot: Pipelines API response'); + }); + + test('Step 13: Cleanup', async () => { + const api = await request.newContext({ baseURL: API }); + if (state.blueprintId) { + await api.delete(`/blueprints/${state.blueprintId}`); + console.log(`Deleted blueprint ${state.blueprintId}`); + } + console.log('Cleanup complete'); + }); +}); diff --git a/env.example b/env.example index 19acb7c94af..04e0ab0a5ed 100755 --- a/env.example +++ b/env.example @@ -34,10 +34,14 @@ SKIP_SUBTASK_PROGRESS=false PORT=8080 MODE=release -# PUSH_API_ALLOWED_TABLES=table1,table2 +# Push API is disabled by default. To enable it, list only application tables +# here. Internal _devlake_* tables are always forbidden. +PUSH_API_ALLOWED_TABLES= NOTIFICATION_ENDPOINT= NOTIFICATION_SECRET= +# Default fallback rate limit for GraphQL clients (used if not dynamically fetched) +GRAPHQL_RATE_LIMIT= API_TIMEOUT=120s API_RETRY=3 API_REQUESTS_PER_HOUR=10000 @@ -88,4 +92,77 @@ SKIP_COMMIT_FILES=true WRAP_RESPONSE_ERROR= # Enable subtasks by default: plugin_name:subtask_name:enabled -ENABLE_SUBTASKS_BY_DEFAULT="jira:collectIssueChangelogs:true,jira:extractIssueChangelogs:true,jira:convertIssueChangelogs:true,tapd:collectBugChangelogs:true,tapd:extractBugChangelogs:true,tapd:convertBugChangelogs:true,zentao:collectBugRepoCommits:true,zentao:extractBugRepoCommits:true,zentao:convertBugRepoCommits:true,zentao:collectStoryRepoCommits:true,zentao:extractStoryRepoCommits:true,zentao:convertStoryRepoCommits:true,zentao:collectTaskRepoCommits:true,zentao:extractTaskRepoCommits:true,zentao:convertTaskRepoCommits:true" \ No newline at end of file +ENABLE_SUBTASKS_BY_DEFAULT="jira:collectIssueChangelogs:true,jira:extractIssueChangelogs:true,jira:convertIssueChangelogs:true,tapd:collectBugChangelogs:true,tapd:extractBugChangelogs:true,tapd:convertBugChangelogs:true,zentao:collectBugRepoCommits:true,zentao:extractBugRepoCommits:true,zentao:convertBugRepoCommits:true,zentao:collectStoryRepoCommits:true,zentao:extractStoryRepoCommits:true,zentao:convertStoryRepoCommits:true,zentao:collectTaskRepoCommits:true,zentao:extractTaskRepoCommits:true,zentao:convertTaskRepoCommits:true" + +########################## +# OIDC / Authentication +########################## +# Master switch. Auth is disabled by default; set true to require +# authentication. When enabled without OIDC, DevLake accepts API keys for +# /rest/* and can trust X-Forwarded-User from an upstream proxy. +AUTH_ENABLED=false + +# OIDC user login. Requires AUTH_ENABLED=true. +OIDC_ENABLED=false + +# Shared secret required before DevLake trusts X-Forwarded-User and +# X-Forwarded-Email from an upstream proxy. Configure the proxy to send the +# same value as X-Forwarded-User-Secret on each forwarded request. +FORWARDED_USER_SECRET= + +# Comma-separated provider identifiers. Each name binds to the env +# vars OIDC__ISSUER_URL, OIDC__CLIENT_ID, etc. Add a name and a +# matching block of vars to onboard another IdP. +# Example: OIDC_PROVIDERS=entra,google +OIDC_PROVIDERS= + +# Per-provider config. Replicate the OIDC_ENTRA_* block under a different +# prefix for each name listed in OIDC_PROVIDERS. +# Microsoft Entra ID example: https://login.microsoftonline.com//v2.0 +OIDC_ENTRA_ISSUER_URL= +OIDC_ENTRA_CLIENT_ID= +OIDC_ENTRA_CLIENT_SECRET= +# Must match the redirect URI registered with the IdP. The path is the same +# for every provider; the state cookie disambiguates which one comes back. +# Devcontainer dev: http://localhost:4000/api/auth/callback +OIDC_ENTRA_REDIRECT_URL= +# Comma-separated. `openid` is required. +OIDC_ENTRA_SCOPES=openid,profile,email +# Label rendered on the UI login button. +OIDC_ENTRA_DISPLAY_NAME=Entra ID +# Authenticate the code exchange with an Azure Workload Identity federated +# assertion (read from the SA token file injected by the workload-identity +# webhook) instead of OIDC_ENTRA_CLIENT_SECRET. Requires the pod label +# `azure.workload.identity/use: "true"` and a federated credential on the +# Entra App Registration. Entra-only. +OIDC_ENTRA_USE_WORKLOAD_IDENTITY=false + +# Google example — create an OAuth 2.0 Web client at console.cloud.google.com +# (APIs & Services → Credentials). Configure the OAuth consent screen first +# and add yourself as a test user while the app is in Testing status. +OIDC_GOOGLE_ISSUER_URL=https://accounts.google.com +OIDC_GOOGLE_CLIENT_ID= +OIDC_GOOGLE_CLIENT_SECRET= +OIDC_GOOGLE_REDIRECT_URL= +OIDC_GOOGLE_SCOPES=openid,profile,email +OIDC_GOOGLE_DISPLAY_NAME=Google + +# When true, /auth/logout returns the IdP's end_session_endpoint so the UI +# can also sign the user out at the IdP. +OIDC_LOGOUT_REDIRECT=false + +# Required when OIDC_ENABLED=true. At least 32 bytes of high-entropy data. +# Used to sign session JWTs (HS256) and to derive the AES-GCM key that +# encrypts the OIDC state cookie. Rotating this invalidates all sessions. +SESSION_SECRET= +# How long a session cookie is valid. Format: any time.ParseDuration value. +SESSION_TTL=8h +# Leave empty for host-only cookies. Set when serving the API and UI from +# different subdomains of the same parent (e.g. .example.com). +COOKIE_DOMAIN= +# Set to false ONLY for local HTTP development. +COOKIE_SECURE=true + +# Restrict OIDC logins to specific users or domains (comma-separated, case-insensitive) +OIDC_ALLOW_EMAILS= +OIDC_ALLOW_DOMAINS= diff --git a/grafana/Dockerfile b/grafana/Dockerfile index 406e7c9a09f..cd401b9bb5e 100644 --- a/grafana/Dockerfile +++ b/grafana/Dockerfile @@ -23,19 +23,23 @@ #While incubation status is not necessarily a reflection of the completeness or stability of the code, #it does indicate that the project has yet to be fully endorsed by the ASF. -FROM grafana/grafana:11.6.2 +FROM grafana/grafana:13.0.2 COPY ./provisioning/dashboards /etc/grafana/provisioning/dashboards -COPY ./provisioning/datasources /etc/grafana/provisioning/datasources COPY ./dashboards /etc/grafana/dashboards +COPY ./scripts/entrypoint.sh /entrypoint.sh COPY ./img/grafana_icon.svg /usr/share/grafana/public/img/grafana_icon.svg COPY ./img /usr/share/grafana/public/img/lake ENV GF_USERS_ALLOW_SIGN_UP=false ENV GF_SERVER_SERVE_FROM_SUB_PATH=true ENV GF_DASHBOARDS_JSON_ENABLED=true ENV GF_LIVE_ALLOWED_ORIGINS='*' -ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/dashboards/Homepage.json +ENV GF_SERVER_PROTOCOL=http +ENV GF_SERVER_DOMAIN=localhost:3000 +ENV GF_SERVER_ROOT_URL=http://localhost:3000/grafana USER root -RUN grafana-cli plugins install grafana-piechart-panel -RUN chgrp -R 0 /etc/grafana /usr/share/grafana /var/lib/grafana && \ +# grafana-piechart-panel removed: dashboards use the core "piechart" panel type +RUN chmod +x /entrypoint.sh && \ + chgrp -R 0 /etc/grafana /usr/share/grafana /var/lib/grafana && \ chmod -R g=u /etc/grafana /usr/share/grafana /var/lib/grafana USER 101 +ENTRYPOINT ["/entrypoint.sh"] diff --git a/grafana/_archive/Daily_Needs_To_Be_Improved.json b/grafana/_archive/Daily_Needs_To_Be_Improved.json deleted file mode 100644 index 7cbeec9924a..00000000000 --- a/grafana/_archive/Daily_Needs_To_Be_Improved.json +++ /dev/null @@ -1,389 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "", - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 32, - "iteration": 1661321745952, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 57, - "options": { - "content": "- Use Cases: This dashboard can be used to track bugs.\n- Data Source Required: GitHub ([transformation](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional) required. Additional settings are required to get version data).", - "mode": "markdown" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "title": "Dashboard Introduction", - "type": "text" - }, - { - "datasource": "mysql", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "label_name" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "count" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "palette-classic" - } - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 4 - }, - "id": 59, - "options": { - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "hidden", - "placement": "right", - "values": [ - "value", - "percent" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": true - }, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "select \ndistinct il.label_name, count(il.label_name) as count\nfrom \nissues i\nleft join issue_labels il on i.id = il.issue_id\nwhere i.type = \"BUG\" and il.label_name like \"%O-%\" and i.updated_date > \"2021-08-01 00:00:00.000\"\ngroup by il.label_name\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "id" - ], - "type": "column" - } - ] - ], - "table": "ae_projects", - "timeColumn": "ae_create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "01. Bug Source", - "type": "piechart" - }, - { - "datasource": "mysql", - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "displayMode": "auto" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "=avg_bug_age" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "issue_key" - }, - "properties": [ - { - "id": "custom.width", - "value": 81 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "title" - }, - "properties": [ - { - "id": "custom.width", - "value": 535 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "created_date" - }, - "properties": [ - { - "id": "custom.width", - "value": 149 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "queue_time_in_days" - }, - "properties": [ - { - "id": "custom.width", - "value": 140 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "issue_url" - }, - "properties": [ - { - "id": "custom.width", - "value": 399 - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "sha" - }, - "properties": [ - { - "id": "custom.width", - "value": 328 - } - ] - } - ] - }, - "gridPos": { - "h": 6, - "w": 17, - "x": 7, - "y": 4 - }, - "id": 61, - "options": { - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with issue_sha_deveq as(\nselect\ni.url as issue_url, c.sha, c.dev_eq\nfrom \npull_request_issues pri\nleft join pull_request_commits prc on prc.pull_request_id = pri.pull_request_id\nleft join issues i on pri.issue_id = i.id\nleft join commits c on c.sha = prc.commit_sha\nwhere i.type = \"BUG\" and i.status = \"DONE\" and c.sha != \"\" and c.committed_date > \"2021-08-01 00:00:00.000\"\n)\n\nselect\nissue_url, sum(dev_eq) as dev_eq_total\nfrom\nissue_sha_deveq\ngroup by issue_url \norder by dev_eq_total DESC", - "refId": "A", - "select": [ - [ - { - "params": [ - "id" - ], - "type": "column" - } - ] - ], - "table": "ae_projects", - "timeColumn": "ae_create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "02. Bug Dev_eq", - "type": "table" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": null, - "definition": "select concat(name, '--', id) as text from repos", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Repo", - "multi": true, - "name": "repo_id", - "options": [], - "query": "select concat(name, '--', id) as text from repos", - "refresh": 1, - "regex": "/^(?.*)--(?.*)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Daily_Needs_To_Be_Improved", - "uid": "2xuOaQUnk5", - "version": 25 -} \ No newline at end of file diff --git a/grafana/_archive/DataSourceSpecificDashboardsHomepage.json b/grafana/_archive/DataSourceSpecificDashboardsHomepage.json deleted file mode 100644 index ce589d087b2..00000000000 --- a/grafana/_archive/DataSourceSpecificDashboardsHomepage.json +++ /dev/null @@ -1,293 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 12, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 0, - "y": 0 - }, - "id": 10, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"jira\"\n

Jira

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 7, - "y": 0 - }, - "id": 5, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"gitlab\"\n

Gitlab

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "datasource": null, - "gridPos": { - "h": 18, - "w": 10, - "x": 14, - "y": 0 - }, - "id": 9, - "links": [], - "options": { - "content": "![Merico Logo](/public/img/lake/logo.png \"Merico\")\n***\n# **MARI Introduction**\n***\nMARI is a methodology developed specifically to bring the process of data driven software engineering to organizations of all sizes in a simple and organized way; empowering them to translate measurement into insight, and insight into results. MARI is a cycle of iterations, focused on mapping challenges to metrics, metrics to actions, and actions to improvements. \n

\n\n\n#### *M (Measure)*\n \nGreat answers require great questions. To get started, identify specific and clearly definable pain points in your development processes or teams, set tightly constrained goals, and then with team consensus, prioritize those goals. From here, identify 'North Star' metrics and measurements that map to each goal. Once you have clearly defined and documented the pain-points, mapped them to metrics, and prioritized, it's time to standardize these metrics and methods across teams. \n\n##### *A (Analyze)*\n\nEffective measures are only as good as the analysis that gets applied to them. To understand productivity in an effective and quantitative way, look towards key attributes like trend, distribution, variability, dispersion, and periodicity. These analysis techniques can uncover problems and opportunities. \n\n##### *R (Review)*\n\nOnce you have equipped yourself with the results of the analysis, it's time to coherently review them and discover what hinders your team from reaching full potential. With key stakeholders in the conversation, assess your workflow, toolset, developer satisfaction and many other aspects with an eye towards identifying cause and effect. It is critical that this part of the process is focused on the real root causes, not blame. Review should involve those affected, and those with the responsibility. \n\n##### *I (Improve)*\n\nAfter review, it's time to prepare a defined and clear iteration. What actions can be taken to potentially address the areas of opportunity? These actions should be clear, defined, measurable, and focused enough that success or failure can be clearly identified and attributed. In larger organizations, consider A/B testing the action plan between a few teams, and be sure to test and assess action plans at smaller scale before implementing enterprise-wide. \n \nAs you implement action plans for improvement, refresh your MARI cycle to coherently drive progress and evolution.\n\n", - "mode": "markdown" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 0, - "y": 9 - }, - "id": 13, - "interval": null, - "links": [], - "options": { - "content": " \n
\n \"jenkins\"\n

Jenkins

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "description": "", - "gridPos": { - "h": 9, - "w": 7, - "x": 7, - "y": 9 - }, - "id": 12, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"requirement\"\n

New Dashboard

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "transparent": true, - "type": "text" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Data Source Specific Dashboards-Homepage", - "uid": "Lv1XbLHnk", - "version": 2 -} diff --git a/grafana/_archive/DeliveryCapability(RequireJenkinsData).json b/grafana/_archive/DeliveryCapability(RequireJenkinsData).json deleted file mode 100644 index 36aa2d01e9f..00000000000 --- a/grafana/_archive/DeliveryCapability(RequireJenkinsData).json +++ /dev/null @@ -1,470 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 9, - "iteration": 1637051222519, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": true, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/RXJZNpMnz/user-value-specific-dashboards-homepage?orgId=1" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "user_value" - ], - "targetBlank": false, - "title": "metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "datasource": "mysql", - "description": "1. Number of builds executed.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n count(*)\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Count", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "mysql", - "description": "The percentage of successful, failed, and aborted builds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "decimals": 0, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 0 - }, - "id": 37, - "interval": null, - "links": [], - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "values": [ - "percent", - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": true - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n result as metric,\n count(*) as value\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nGROUP BY 1,2\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Result Distribution", - "type": "piechart" - }, - { - "datasource": "mysql", - "description": "Number of successful builds / Number of total builds", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n 1.0 * count(case when result = 'SUCCESS' then 1 else null end)/count(*)\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Success Rate", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "unit": "percentunit" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 6 - }, - "hiddenSeries": false, - "id": 50, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _build_success_rate as(\r\n SELECT\r\n DATE_ADD(date(start_time), INTERVAL -$interval(date(start_time))+1 DAY) as time,\r\n result\r\n FROM\r\n jenkins_builds\r\n WHERE\r\n $__timeFilter(start_time)\r\n)\r\n\r\nSELECT \r\n timestamp(time) as time,\r\n 1.0 * sum(case when result = 'SUCCESS' then 1 else 0 end)/ count(*) as \"Build Success Rate\"\r\nFROM _build_success_rate\r\ngroup by 1\r\norder by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Build Success Rate over Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:262", - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:263", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "capability", - "user_value" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "text": "Month", - "value": "DAY" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Time Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": false, - "text": "Week", - "value": "DAYOFWEEK" - }, - { - "selected": true, - "text": "Month", - "value": "DAY" - } - ], - "query": "Week : DAYOFWEEK, Month : DAY", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Delivery Capability (require Jenkins data)", - "uid": "3Lv1ImSnk", - "version": 5 -} diff --git a/grafana/_archive/DeliveryCost(RequireGitlabData).json b/grafana/_archive/DeliveryCost(RequireGitlabData).json deleted file mode 100644 index b5faba8349e..00000000000 --- a/grafana/_archive/DeliveryCost(RequireGitlabData).json +++ /dev/null @@ -1,408 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 1, - "iteration": 1637051230516, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": true, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/RXJZNpMnz/user-value-specific-dashboards-homepage?orgId=1" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "user_value" - ], - "targetBlank": false, - "title": "metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "datasource": "mysql", - "description": "1. Number of people who have created or reviewed a Pull/Merge Request.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with all_developers as(\n select \n gitlab_reviewers.name as user_name\n from gitlab_merge_requests\n join gitlab_reviewers on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\n\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_merge_requests.project_id = $repo_id\n union\n select \n distinct author_username as user_name \n from gitlab_merge_requests\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_merge_requests.project_id = $repo_id\n union\n select \n distinct author_name as user_name \n from gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and gitlab_commits.project_id = $repo_id\n)\n\n\nSELECT\n now() AS \"time\",\n count(distinct user_name) as developer_count\nFROM all_developers", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Developer Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Total number of Commit Authors.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n count(distinct author_name) as value\nFROM gitlab_commits\nWHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Author Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of Pull/Merge Request Reviewers.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 6 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \n now() as time,\n count(distinct name)\nfrom\n gitlab_reviewers", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Reviewer Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Pull Request Reviewer Count/Total Developer Count", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0.2 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 6 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with all_user_names as(\n select \n distinct gitlab_reviewers.name as user_name\n from gitlab_reviewers\n join gitlab_merge_requests on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_reviewers.project_id = $repo_id\n union\n select \n distinct author_username as user_name \n from gitlab_merge_requests\n WHERE\n $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n union\n select \n distinct author_name as user_name \n from gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\n),\n\nreviewer as (\n select distinct gitlab_reviewers.name from gitlab_reviewers\n join gitlab_merge_requests on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\nWHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_reviewers.project_id = $repo_id\n)\n\nSELECT\n now() AS \"time\",\n 1.0*count(distinct reviewer.name)/count(distinct user_name) as value\nFROM reviewer, all_user_names", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Reviewer Rate (%)", - "type": "stat" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "cost", - "user_value" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "mysql", - "definition": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Choose Repo", - "multi": false, - "name": "repo_id", - "options": [], - "query": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "refresh": 1, - "regex": "/^(?[^:]+): (?\\d+)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Delivery Cost (require Gitlab data)", - "uid": "TV-BnpM7k", - "version": 7 -} diff --git a/grafana/_archive/DeliveryQuality(RequireJiraAndGitlabData).json b/grafana/_archive/DeliveryQuality(RequireJiraAndGitlabData).json deleted file mode 100644 index 92e2fef5ee0..00000000000 --- a/grafana/_archive/DeliveryQuality(RequireJiraAndGitlabData).json +++ /dev/null @@ -1,1699 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 2, - "iteration": 1637051238473, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/RXJZNpMnz/user-value-specific-dashboards-homepage?orgId=1" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "user_value" - ], - "targetBlank": false, - "title": "Metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 46, - "interval": null, - "links": [], - "options": { - "content": "
\n
\n \"R&D\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Charts are organized by the Research & Development stages below", - "transparent": true, - "type": "text" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 6 - }, - "id": 48, - "options": { - "content": "
\n
\n \"No.1\"\n

Development

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of Pull/Merge request created.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 10 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM \n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Total number of Pull/Merge request merged.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 10 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n count(*) as value\nFROM\n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n jbgp.jira_board_id = $board_id\n and state = 'merged'\n and $__timeFilter(gitlab_created_at)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Pass Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Pull Request Pass Count/Pull Request Count", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 10 - }, - "id": 37, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n 1.0 * count(case when state = 'merged' then 1 else null end)/count(*)\nFROM\n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Pass Rate (%)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Pull Request Pass Rate over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", value \"pull_request_pass_rate\" of \"2021-06-01\" calculates the PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _mr_request_rate as(\n SELECT\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n state\n FROM\n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\n WHERE\n jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\n)\n\nSELECT \n timestamp(time) as time,\n 1.0 * count(case when state = 'merged' then 1 else null end)/count(*) as \"Pull Request Pass Rate\"\nFROM _mr_request_rate\nGROUP BY 1\nORDER BY 1\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Pass Rate over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The average round of PR/MR review.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 22 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n avg(review_rounds) as value\nFROM \n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n state = 'merged'\n and review_rounds > 0\n and jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Pull Request Review Round", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Avg PR/MR Review Round over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", value \"Avg Pull Request Review Round\" of \"2021-06-01\" calculates the PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 22 - }, - "id": 55, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n timestamp(DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY)) as time,\n avg(review_rounds) as \"Pull Request Reveiw Round\"\nFROM\n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n state = 'merged'\n and review_rounds > 0\n and jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\nGROUP BY 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Pull Request Review Round over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The average round of PR/MR review of PR authors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 28 - }, - "id": 54, - "options": { - "barWidth": 0.52, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n author_username as Author,\n avg(review_rounds) as value\nFROM \n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\nWHERE\n state = 'merged'\n and review_rounds > 0\n and jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\nGROUP BY 1,2\nORDER BY 3 desc\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Pull Request Review Round by Author", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "One-time pass rate for PR over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "One-time Pass Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 56, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _review_rounds as(\n SELECT\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n review_rounds\n FROM\n gitlab_merge_requests gmr\n LEFT JOIN jira_board_gitlab_projects jbgp ON jbgp.gitlab_project_id = gmr.project_id\n WHERE\n state = 'merged'\n and review_rounds > 0\n and jbgp.jira_board_id = $board_id\n and $__timeFilter(gitlab_created_at)\n)\n\nSELECT \n timestamp(time) as time,\n count(case when review_rounds = 1 then true else null end)/count(*) as one_time_pass_rate\nFROM \n _review_rounds\nGROUP BY 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "One-time Pass Rate for Code Reviews over Time", - "type": "timeseries" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 40 - }, - "id": 50, - "options": { - "content": "
\n
\n \"No.2\"\n

Test

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of bugs created.\n2. The bugs being calculated are filtered by \"bug creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 44 - }, - "id": 16, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n count(*) as value\nFROM\n jira_issues ji\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\nWHERE\n std_type = 'Bug'\n and jbi.board_id = $board_id\n and $__timeFilter(created)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Number of Bugs found in Testing", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of bugs created over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", bug_count of \"2021-06-01\" calculates the bugs whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bug Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 44 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _bugs as (\n SELECT\n DATE_ADD(date(ji.created), INTERVAL -$interval(date(ji.created))+1 DAY) as time,\n count(distinct ji.issue_id) as bug_count\n FROM\n jira_issues ji\n LEFT JOIN jira_board_issues jbi ON jbi.issue_id = ji.issue_id\n WHERE\n std_type = 'Bug'\n and jbi.board_id = $board_id\n and $__timeFilter(ji.created)\n group by 1\n order by 1\n) \n\nselect \n timestamp(time) as time,\n bug_count\nfrom _bugs", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Number of Bugs found in Testing over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "1. Total number of bugs found per 1,000 lines of code, including both added and deleted llines of code.\n2. The bugs being calculated are filtered by \"bug creation time\". The lines of code being calculated are filtered by \"commit creation time\" (both are affected by the time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 0.32 - }, - { - "color": "red", - "value": 2.39 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 50 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with bc as (\n\tselect \n\t count(*) as bug_count\n\tfrom \n\t jira_issues ji \n\t left join jira_board_issues jbi on (jbi.issue_id = ji.issue_id)\n\twhere \n\t jbi.board_id = $board_id\n\t and ji.std_type = 'Bug'\n\t and $__timeFilter(ji.created)\n),\n\nloc as (\n\tselect \n\t sum(additions + deletions) as line_count\n\tfrom \n\t gitlab_commits gc \n\t left join jira_board_gitlab_projects jbgp on jbgp.gitlab_project_id = gc.project_id\n\twhere\n\t jbgp.jira_board_id = $board_id\n\t and gc.title not like 'Merge branch %'\n\t and $__timeFilter(authored_date)\n)\n\nselect \n now() as time, \n 1.0 * bc.bug_count / loc.line_count * 1000 as value\nfrom loc,bc\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Bug Count per 1k lines of code (Total)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Bug Count per 1k lines of code over time.\n2. When Time Interval is set to \"month\", bug_count_per_1k_LOC of \"2021-06-01\" calculates the bugs whose creation time falls under [2020-06-01, 2020-07-01) and lines of code whose commit creation time falls under [2020-06-01, 2020-07-01) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bug Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 50 - }, - "id": 22, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with bc as (\n\tselect\n\t DATE_ADD(date(ji.created), INTERVAL -$interval(date(ji.created))+1 DAY) as time,\n\t count(*) as bug_count\n\tfrom \n\t jira_issues ji \n\t left join jira_board_issues jbi on jbi.issue_id = ji.issue_id\n\twhere \n\t jbi.board_id = $board_id\n\t and ji.type = 'Bug'\n\t and $__timeFilter(ji.created)\n\tgroup by 1\n),\nloc as (\n\tselect \n\t DATE_ADD(date(gc.authored_date), INTERVAL -$interval(date(gc.authored_date))+1 DAY) as time,\n\t sum(additions + deletions) as line_count\n\tfrom \n\t gitlab_commits gc \n\t left join jira_board_gitlab_projects jbgp on jbgp.gitlab_project_id = gc.project_id\n\twhere \n\t jbgp.jira_board_id = $board_id\n\t and gc.title not like 'Merge branch %'\n\t and $__timeFilter(authored_date)\n\tgroup by 1\n),\nbug_count_per_1k_loc as(\n select \n bc.time,\n 1.0 * bc.bug_count / loc.line_count * 1000 as bug_count_per_1k_loc\n from \n loc\n left join bc on bc.time = loc.time\n where\n bc.bug_count is not null \n and loc.line_count is not null\n)\n\nselect \n timestamp(time) as time, \n bug_count_per_1k_loc \nfrom bug_count_per_1k_loc\norder by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Bug Count per 1k lines of code (Breakdown by Time)", - "type": "timeseries" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 56 - }, - "id": 52, - "options": { - "content": "
\n
\n \"No.3\"\n

Operation

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of incidents created.\n2. The incidents being calculated are filtered by \"incident creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 60 - }, - "id": 38, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n count(*) as value\nFROM\n jira_issues ji\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\nWHERE\n std_type = 'Incident'\n and jbi.board_id = $board_id\n and $__timeFilter(created)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Number of Incidents found after shipping", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of incidents created over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", incident_count of \"2021-06-01\" calculates the incidents whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Incident Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 60 - }, - "id": 39, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _incidents as (\n SELECT\n DATE_ADD(date(ji.created), INTERVAL -$interval(date(ji.created))+1 DAY) as time,\n count(distinct ji.issue_id) as incident_count\n FROM\n jira_issues ji\n LEFT JOIN jira_board_issues jbi ON jbi.issue_id = ji.issue_id\n WHERE\n std_type = 'Incident'\n and jbi.board_id = $board_id\n and $__timeFilter(ji.created)\n group by 1\n order by 1\n) \n\nselect \n timestamp(time) as time,\n incident_count\nfrom _incidents", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Number of Incidents found after shipping over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "1. Total number of incidents found per 1,000 lines of code, including both added and deleted llines of code.\n2. The incidents being calculated are filtered by \"incident creation time\". The lines of code being calculated are filtered by \"commit creation time\" (both are affected by the time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "#EAB839", - "value": 0.32 - }, - { - "color": "red", - "value": 2.39 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 66 - }, - "id": 28, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with ic as (\n\tselect \n\t count(*) as incident_count\n\tfrom \n\t jira_issues ji \n\t left join jira_board_issues jbi on (jbi.issue_id = ji.issue_id)\n\twhere \n\t jbi.board_id = $board_id\n\t and ji.type = 'Incident'\n\t and $__timeFilter(ji.created)\n),\n\nloc as (\n\tselect \n\t sum(additions + deletions) as line_count\n\tfrom \n\t gitlab_commits gc \n\t left join jira_board_gitlab_projects jbgp on jbgp.gitlab_project_id = gc.project_id\n\twhere\n\t jbgp.jira_board_id = $board_id\n\t and gc.title not like 'Merge branch %'\n\t and $__timeFilter(authored_date)\n)\n\nselect \n now() as time, \n 1.0 * ic.incident_count / loc.line_count * 1000 as value\nfrom loc,ic", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Incident Count per 1k lines of code (Total)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Incident Count per 1k lines of code over time.\n2. When Time Interval is set to \"month\", incident_count_per_1k_LOC of \"2021-06-01\" calculates the incidents whose creation time falls under [2020-06-01, 2020-07-01) and lines of code whose commit creation time falls under [2020-06-01, 2020-07-01) ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Incident Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 66 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with ic as (\n\tselect \n\t DATE_ADD(date(ji.created), INTERVAL -$interval(date(ji.created))+1 DAY) as time,\n\t count(*) as incident_count\n\tfrom \n\t jira_issues ji \n\t left join jira_board_issues jbi on jbi.issue_id = ji.issue_id\n\twhere \n\t jbi.board_id = $board_id\n\t and ji.type = 'Incident'\n\t and $__timeFilter(ji.created)\n\tgroup by 1\n),\nloc as (\n\tselect \n\t DATE_ADD(date(gc.authored_date), INTERVAL -$interval(date(gc.authored_date))+1 DAY) as time,\n\t sum(additions + deletions) as line_count\n\tfrom \n\t gitlab_commits gc \n\t left join jira_board_gitlab_projects jbgp on jbgp.gitlab_project_id = gc.project_id\n\twhere \n\t jbgp.jira_board_id = $board_id\n\t and gc.title not like 'Merge branch %'\n\t and $__timeFilter(authored_date)\n\tgroup by 1\n),\nincident_count_per_1k_loc as(\n select \n ic.time,\n 1.0 * ic.incident_count / loc.line_count * 1000 as incident_count_per_1k_loc\n from \n loc\n left join ic on ic.time = loc.time\n where\n ic.incident_count is not null \n and loc.line_count is not null\n)\n\nselect \n timestamp(time) as time, \n incident_count_per_1k_loc \nfrom incident_count_per_1k_loc\norder by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Incident Count per 1k lines of code (Breakdown by Time)", - "type": "timeseries" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "quality", - "user_value" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "Month", - "value": "DAY" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Time Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": false, - "text": "Week", - "value": "DAYOFWEEK" - }, - { - "selected": true, - "text": "Month", - "value": "DAY" - } - ], - "query": "Week : DAYOFWEEK, Month : DAY", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "mysql", - "definition": "select distinct concat(name, ': ', board_id) from jira_boards", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Choose Board", - "multi": false, - "name": "board_id", - "options": [], - "query": "select distinct concat(name, ': ', board_id) from jira_boards", - "refresh": 1, - "regex": "/^(?[^:]+): (?\\d+)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Delivery Quality (require Jira and Gitlab data)", - "uid": "nUC-7tGnk", - "version": 14 -} diff --git a/grafana/_archive/DeliveryVelocity(RequireJiraAndGitlabData).json b/grafana/_archive/DeliveryVelocity(RequireJiraAndGitlabData).json deleted file mode 100644 index 8df95d940af..00000000000 --- a/grafana/_archive/DeliveryVelocity(RequireJiraAndGitlabData).json +++ /dev/null @@ -1,4093 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 3, - "iteration": 1637051247070, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": true, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/RXJZNpMnz/user-value-specific-dashboards-homepage?orgId=1" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "user_value" - ], - "targetBlank": false, - "title": "Metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 5, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 89, - "interval": null, - "links": [], - "options": { - "content": "
\n
\n \"R&D\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Charts are organized by the Research & Development stages below", - "transparent": true, - "type": "text" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 76, - "options": { - "content": "
\n
\n \"Cross-stage\"\n

Cross-stage Metrics

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. The average lead time of requirements.\n2. Lead time refers to the length of time from requirement creation to delivery.\n3. The requirements being calculated are filtered by \"requirement resolution time\" (time filter at the upper-right corner) and \"Jira board\"(\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 14 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 9 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^value$/", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \r\n now() as time,\r\n avg(lead_time/1440) as value\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Requirement'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\ngroup by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Requirement Lead Time (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The lead time at which 80% requirements‘ lead time lies below it.\n2. The requirements being calculated are filtered by \"requirement resolution time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 21 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 9 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n ji.lead_time,\r\n percent_rank() over (order by lead_time asc) as ranks\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect\r\n now() as time,\r\n max(lead_time/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8\r\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "80th Percentile Requirement Lead Time (day)", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. The average requirement lead time over time.\n2. The time granularity can be switched to Week or Month by \"Time Interval\" above. \n3. When Time Interval is set to \"Month\", the average_lead_time of \"2021-06-01\" refers to the average lead time of requirements whose resolution time falls under [2020-06-01, 2020-07-01).", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 9 - }, - "hiddenSeries": false, - "id": 17, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select \r\n DATE_ADD(date(resolution_date), INTERVAL -$interval(date(resolution_date))+1 DAY) as time,\r\n lead_time\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n timestamp(time) as time,\r\n avg(lead_time/1440) as average_lead_time\r\nfrom _requirement\r\ngroup by 1\r\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Requirement Lead Time over Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:242", - "format": "short", - "label": "Lead Time(day)", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:243", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", - "fill": 0, - "fillGradient": 4, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 15 - }, - "hiddenSeries": false, - "id": 15, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 8, - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n round(ji.lead_time/1440) as lead_time_day\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [ - { - "$$hashKey": "object:469", - "colorMode": "ok", - "fill": true, - "line": true, - "op": "lt", - "value": 0.8, - "yaxis": "right" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Requirement Lead Time (Cumulative Distribution)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "current" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:76", - "format": "percentunit", - "label": "Percent Rank (%)", - "logBase": 1, - "max": "1.2", - "min": null, - "show": true - }, - { - "$$hashKey": "object:77", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "mysql", - "description": "The average lead time of requirements under different Jira Epics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Lead Time (day)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 21 - }, - "id": 18, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n ji.issue_id,\r\n epic_key,\r\n ji.lead_time\r\n from\r\n jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n ji.std_type = 'Requirement'\r\n and ji.std_status = 'Resolved'\r\n and ji.epic_key is not null\r\n and ji.epic_key != ''\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n now() as time,\r\n epic_key as \"Epic Key\",\r\n avg(lead_time/1440) as value\r\nfrom _requirement\r\ngroup by 1,2\r\norder by 3 desc\r\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Requirement Lead Time by Epic", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "The average time spent in each state of a requirement, including all history states that have been used.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Lead Time(day)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 74, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 27 - }, - "id": 97, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with operation_history as(\n\tSELECT\n\t\tjc.issue_id,\n\t\tjc.created as operation_time,\n\t\tjci.changelog_id,\n\t\tjci.from,\n\t\tjci.from_string,\n\t\tjci.to,\n\t\tjci.to_string,\n\t\tji.created as issue_creation_time,\n\t\tcase \n\t\t\twhen lag(jc.created) over(PARTITION BY jc.issue_id ORDER BY jc.created ASC) is not null then\n\t\t\tTIMESTAMPDIFF(MINUTE, lag(jc.created) over(PARTITION BY jc.issue_id ORDER BY jc.created ASC), jc.created)\n\t\t\telse TIMESTAMPDIFF(MINUTE, ji.created, jc.created) end as time_in_status\n\tFROM\n\t\tjira_changelogs jc\n\t\tJOIN jira_changelog_items jci ON jc.changelog_id = jci.changelog_id\n\t\tJOIN jira_issues ji ON jc.issue_id = ji.issue_id\n\t\tjoin jira_board_issues jbi on ji.issue_id = jbi.issue_id\n\tWHERE\n\t\tjci.field = 'status'\n\t\tand ji.std_type = 'Requirement'\n\t\tand ji.std_status = 'Resolved'\n\t\tand $__timeFilter(ji.resolution_date)\n and jbi.board_id = $board_id\n)\n\nSELECT\n now() as time,\n\tfrom_string as \"Requirement Status\",\n\tavg(time_in_status)/1440 as \"Time in Status\"\nFROM operation_history\nGROUP BY 1,2\nORDER BY 3 desc", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Requirement Lead Time by Status", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "1. Compare the average time spent in each state of a requirement in the last 2 months.\n2. The requirements being calculated are the requirements delivered in the last 2 months.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Lead Time(day)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 74, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 98, - "options": { - "barWidth": 0.9, - "groupWidth": 0.6, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with status_change_history as(\n\tSELECT\n\t\tjc.issue_id,\n\t\tjc.created as operation_time,\n\t\tjci.changelog_id,\n\t\tjci.from,\n\t\tjci.from_string,\n\t\tjci.to,\n\t\tjci.to_string,\n\t\tji.created as issue_creation_time,\n\t\tlag(jc.created) over(PARTITION BY jc.issue_id ORDER BY jc.created ASC) as last_operation_time,\n\t\tcase \n\t\t\twhen lag(jc.created) over(PARTITION BY jc.issue_id ORDER BY jc.created ASC) is not null then\n\t\t\tTIMESTAMPDIFF(MINUTE, lag(jc.created) over(PARTITION BY jc.issue_id ORDER BY jc.created ASC), jc.created)\n\t\t\telse TIMESTAMPDIFF(MINUTE, ji.created, jc.created) end as time_in_status\n\tFROM\n\t\tjira_changelogs jc\n\t\tJOIN jira_changelog_items jci ON jc.changelog_id = jci.changelog_id\n\t\tJOIN jira_issues ji ON jc.issue_id = ji.issue_id\n\t\tjoin jira_board_issues jbi on ji.issue_id = jbi.issue_id\n\tWHERE\n\t\tjci.field = 'status'\n\t\tand ji.std_type = 'Requirement'\n\t\tand ji.std_status = 'Resolved'\n\t\tand $__timeFilter(ji.resolution_date)\n and jbi.board_id = $board_id\n),\n\ntime_in_status as(\n\tSELECT \n\t\tDATE_ADD(date(issue_creation_time), INTERVAL -DAY(date(issue_creation_time))+1 DAY) as time,\n\t\tfrom_string,\n\t\tavg(time_in_status)/60 as time_in_status\n\tFROM status_change_history\n\tGROUP BY 1,2\n),\n\nlast_month as(\n\tselect \n\t\tDATE_ADD(date(CURDATE()), INTERVAL -DAY(date(CURDATE()))+1 DAY) as last_month\n),\n\nmonth_before_last_month as(\n\tSELECT \n\t\tDATE_ADD(DATE_ADD(date(CURDATE()), INTERVAL -DAY(date(CURDATE()))+1 DAY), INTERVAL -1 MONTH) as two_month_before\n),\n\n\nlast_month_record as(\n\tSELECT\n\t\tfrom_string as issue_status,\n\t\tcase when time in (SELECT last_month from last_month) then time_in_status else null end as last_month\n\tfrom time_in_status\n\tWHERE \n\t\ttime in (SELECT last_month from last_month)\n),\n\ntwo_month_before_record as(\n\tSELECT\n\t\tfrom_string as issue_status,\n\t\tcase when time in (SELECT two_month_before from month_before_last_month) then time_in_status else null end as two_month_before\n\tfrom time_in_status\n\tWHERE \n\t\ttime in (SELECT two_month_before from month_before_last_month)\n)\n\nSELECT\n\tnow() as time,\n\tmbr.two_month_before AS \"The Month Before Last\",\n\tlmr.last_month AS \"Last Month\",\n\tlmr.issue_status\nFROM last_month_record lmr\nleft join two_month_before_record mbr on lmr.issue_status = mbr.issue_status ", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Requirement Lead Time by Status in Last 2 Months", - "type": "barchart" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 39 - }, - "id": 92, - "options": { - "content": "
\n
\n \"No.1\"\n

Requirement

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of requirements created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 43 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "select \r\n now() as time,\r\n count(*) as value\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Requirement'\r\n and created is not null \r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\ngroup by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Requirement Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The number of requirements created over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", the requirement_count of \"2021-06-01\" refers to the number of requirements whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 18, - "x": 6, - "y": 43 - }, - "id": 6, - "interval": null, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n ji.issue_id AS id,\r\n DATE_ADD(date(created), INTERVAL -$interval(date(created))+1 DAY) as time\r\n from\r\n jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and created is not null \r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect\r\n timestamp(time) as time,\r\n count(distinct id) as requirement_count\r\nfrom _requirement\r\ngroup by 1\r\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requirement Count over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "1. The number of requirements under different Jira Epics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "No. of Requirement", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 49 - }, - "id": 7, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n ji.issue_id,\r\n ji.epic_key\r\n from\r\n jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and created is not null\r\n and epic_key is not null\r\n and epic_key != ''\r\n and std_status = 'Resolved'\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n now() as time,\r\n epic_key as \"Epic Key\",\r\n count(*) as value\r\nfrom _requirement\r\ngroup by 1,2\r\norder by 3 desc\r\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requirement Count by Epic", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "1. Number of requirements that have been delivered.\n2. A requirement is delivered when its status equals 'delivered'.\n3. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 55 - }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "select\r\n now() as time,\r\n count(*) as delivered_count\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Requirement'\r\n and ji.std_status = 'Resolved'\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Delivered Requirement Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Delivered Requirement Count/Requirement Count (Total)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 50 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 55 - }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n count(distinct ji.issue_id) as total_count,\r\n count(distinct case when ji.std_status = 'Resolved' then ji.issue_id else null end) as delivered_count\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirement", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Requirement Delivery Rate", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "mysql", - "description": "1. The requirement delivery rate over time.\n2. When Time Interval is set to \"month\", value \"requirement_delivery_rate\" of \"2021-06-01\" calculates the requirements whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Delivery Rate (%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 55 - }, - "id": 23, - "links": [], - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n DATE_ADD(date(created), INTERVAL -$interval(date(created))+1 DAY) as time,\r\n count(distinct ji.issue_id) as total_count,\r\n count(distinct case when ji.std_status = 'Resolved' then ji.issue_id else null end) as delivered_count\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n group by 1\r\n)\r\n\r\n\r\nselect \r\n timestamp(time) as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirement\r\norder by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requirement Delivery Rate over Time", - "type": "timeseries" - }, - { - "cacheTimeout": null, - "datasource": "mysql", - "description": "The average delivery rate of requirements under different Jira Epics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Delivery Rate(%)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 61 - }, - "id": 24, - "links": [], - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select\r\n ji.epic_key as metric,\r\n count(distinct ji.issue_id) as total_count,\r\n count(distinct case when ji.std_status = 'Resolved' then ji.issue_id else null end) as delivered_count\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and epic_key is not null\r\n and epic_key != ''\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n group by 1\r\n)\r\n\r\n\r\nselect \r\n now() as time,\r\n metric,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirement\r\norder by 3 desc\r\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Requirement Delivery Rate by Epic", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "1. The average story point (workload) of requirements.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 67 - }, - "id": 93, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "select \r\n now() as time,\r\n avg(std_story_point) as value\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Requirement'\r\n and std_story_point != 0\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\ngroup by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Avg Requirement Granunarity(Story Point)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The story point/estimated hours at which 80% requirements' story point/estimated hours lies below it.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 67 - }, - "id": 94, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n std_story_point,\r\n percent_rank() over (order by std_story_point asc) as ranks\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_story_point != 0\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect\r\n now() as time,\r\n max(std_story_point) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8\r\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "80th Percentile Requirement Granunarity(Story point)", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "The average and 80th percentile requirement granularity over time.", - "fieldConfig": { - "defaults": { - "unit": "short" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 67 - }, - "hiddenSeries": false, - "id": 96, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _requirement as(\r\n select \r\n ji.issue_id,\r\n DATE_ADD(date(resolution_date), INTERVAL -$interval(date(resolution_date))+1 DAY) as time,\r\n std_story_point\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_story_point != 0\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n timestamp(time) as time,\r\n avg(std_story_point) as average_story_point\r\nfrom _requirement\r\ngroup by 1\r\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Requirement Granularity over Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:242", - "format": "short", - "label": "Story Points", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:243", - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. The cumulative distribution of requirement granularity.\n2. Each point refers to the percent rank of a value of granularity.", - "fill": 0, - "fillGradient": 4, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 73 - }, - "hiddenSeries": false, - "id": 95, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 8, - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 5, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n round(std_story_point) as std_story_point\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Requirement'\r\n and std_story_point != 0\r\n and $__timeFilter(created)\r\n and jbi.board_id = $board_id\r\n order by std_story_point asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(std_story_point, 4, ' ') as metric,\r\n percent_rank() over (order by std_story_point asc) as value\r\nfrom _ranks\r\norder by std_story_point asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [ - { - "$$hashKey": "object:469", - "colorMode": "ok", - "fill": true, - "line": true, - "op": "lt", - "value": 0.8, - "yaxis": "right" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Requirement Granularity (Cumulative Distribution)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "transformations": [], - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "current" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:76", - "format": "percentunit", - "label": "Percent Rank (%)", - "logBase": 1, - "max": "1.2", - "min": null, - "show": true - }, - { - "$$hashKey": "object:77", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 79 - }, - "id": 71, - "options": { - "content": "
\n
\n \"No.2\"\n

Development

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Average PR/MR review time.\n2. PR/MR review time refers to the time between the creation time and merged time of a PR/MR.\n3. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 83 - }, - "id": 80, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \n now() as time,\n avg(TIMESTAMPDIFF(MINUTE,gitlab_created_at,merged_at))/1440 as value\nfrom gitlab_merge_requests\nWHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Pull Request Review Time (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The 80th percentile PR/MR review time.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 83 - }, - "id": 83, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\n select \n TIMESTAMPDIFF(SECOND,gitlab_created_at,merged_at)/86400 as metric,\n percent_rank() over (order by TIMESTAMPDIFF(MINUTE,gitlab_created_at,merged_at) asc) as ranks\n from gitlab_merge_requests\n WHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n )\n \nselect\n now() as time,\n max(metric) as value\nfrom _ranks\nwhere \n ranks <= 0.8\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "80th Percentile Pull Request Review Time(day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The average and 80th percentile PR/MR review time over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", the average review time of \"2021-06-01\" refers to the review time of PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Review Time (day)", - "axisPlacement": "auto", - "axisSoftMin": -3, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 82, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with reviewed_mr as(\n select\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n TIMESTAMPDIFF(SECOND,first_comment_time,merged_at)/86400 as review_time\n from gitlab_merge_requests\n WHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n)\n\nselect \n timestamp(time) as time,\n avg(review_time) as \"Average Review Time\"\nfrom reviewed_mr\ngroup by 1\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Review Time over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "1. Total number of commits created.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 89 - }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM gitlab_commits\nWHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of commits created over time.\n2. When Time Interval is set to \"month\", the commit_count\" of \"2021-06-01\" calculates the commits whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Commit Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 6, - "y": 89 - }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _commits as(\n SELECT\n DATE_ADD(date(authored_date), INTERVAL -$interval(date(authored_date))+1 DAY) as time,\n count(*) as commit_count\n FROM gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\n group by 1\n)\n\nSELECT \n timestamp(time) as time,\n commit_count as \"Commit Count\"\nFROM _commits\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Count over Time", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "The number of commits from different repos.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 89 - }, - "hiddenSeries": false, - "id": 37, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n gp.name as metric,\n count(*) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id\nWHERE\n $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1,2\norder by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit Count by Repo", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:89", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:90", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "mysql", - "description": "The number of commits from different commit authors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 95 - }, - "id": 38, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n gc.author_name as \"Author Name\",\n count(*) as value\nFROM gitlab_commits gc \n join gitlab_projects gp on gc.project_id = gp.gitlab_id\nWHERE\n $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1,2\norder by 3 desc\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Count by Author", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "1. Sum of the number of added lines of code under all commits.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 101 - }, - "id": 42, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n sum(additions) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id \nWHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added LOC", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Sum of the number of deleted lines of code under all commits.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 101 - }, - "id": 43, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n sum(deletions) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id \nWHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Deleted LOC", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Added/deleted number of lines of code over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "LOC", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 101 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with loc as(\n SELECT\n DATE_ADD(date(authored_date), INTERVAL -$interval(date(authored_date))+1 DAY) as time,\n sum(additions) as added_LOC,\n sum(deletions) as deleted_LOC\n FROM gitlab_commits\n WHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and project_id = $repo_id\n group by 1\n)\n\nSELECT \n timestamp(time) as time,\n added_LOC as \"Added LOC\",\n deleted_LOC as \"Deleted LOC\"\nFROM loc\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added and Deleted LOC over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The number of added/deleted LOC from different commit authors.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "LOC", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 107 - }, - "id": 49, - "options": { - "barWidth": 0.9, - "groupWidth": 0.6, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n sum(additions) as \"Added LOC\",\n sum(deletions) as \"Removed LOC\",\n gc.author_name\nFROM gitlab_commits gc \n join gitlab_projects gp on gc.project_id = gp.gitlab_id\nWHERE\n author_name is not null\n and message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1,4\norder by 2 desc\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added and Deleted LOC by Author", - "type": "barchart" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 113 - }, - "id": 77, - "options": { - "content": "
\n
\n \"No.3\"\n

Test

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. The average bug age.\n2. Bug age refers to the length of time from bug creation to resolution.\n3. The bugs being calculated are filtered by \"bug resolution time\" (time filter at the upper-right corner) and \"Jira board\"(\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 117 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^value$/", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \r\n now() as time,\r\n avg(lead_time/1440) as value\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Bug'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\ngroup by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Bug Age (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The bug age at which 80% bugs‘ age lies below it.\n2. The bugs being calculated are filtered by \"bug resolution time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 14 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 117 - }, - "id": 65, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n ji.lead_time,\r\n percent_rank() over (order by lead_time asc) as ranks\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Bug'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect\r\n now() as time,\r\n max(lead_time/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8\r\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "80th Percentile Bug Age (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The average and 80th percentile bug age over time.\n2. The time granularity can be switched to Week or Month by \"Time Interval\" above. \n3. When Time Interval is set to \"Month\", the average_bug_age of \"2021-06-01\" refers to the average age of bugs whose resolution time falls under [2020-06-01, 2020-07-01).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bug Age (day)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 117 - }, - "id": 66, - "interval": "", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _bugs as(\r\n select \r\n ji.issue_id,\r\n DATE_ADD(date(resolution_date), INTERVAL -$interval(date(resolution_date))+1 DAY) as time,\r\n lead_time,\r\n round(lead_time/1440) as lead_time_day\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Bug'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n timestamp(time) as time,\r\n avg(lead_time/1440) as \"Average Lead Time\"\r\nfrom _bugs\r\ngroup by 1\r\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Bug Age over Time", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. The cumulative distribution of bug age. \n2. Each point refers to the percent rank of a bug age.", - "fieldConfig": { - "defaults": { - "unit": "percentunit" - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 4, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 123 - }, - "hiddenSeries": false, - "id": 61, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 8, - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n round(ji.lead_time/1440) as lead_time_day\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Bug'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [ - { - "$$hashKey": "object:912", - "colorMode": "ok", - "fill": true, - "line": true, - "op": "lt", - "value": 0.8, - "yaxis": "right" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Bug Age (Cumulative Distribution)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:76", - "format": "percentunit", - "label": "Percent Rank (%)", - "logBase": 1, - "max": "1.2", - "min": null, - "show": true - }, - { - "$$hashKey": "object:77", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "mysql", - "description": "The average time spent on fixing bugs under different Jira Epics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Bug Age(day)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 129 - }, - "id": 59, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _bugs as(\r\n select\r\n ji.issue_id,\r\n ji.epic_key,\r\n ji.lead_time\r\n from\r\n jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Bug'\r\n and std_status = 'Resolved'\r\n and epic_key is not null\r\n and epic_key != ''\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n now() as time,\r\n epic_key as \"Epic Key\",\r\n avg(lead_time)/1440 as \"Bug Age\"\r\nfrom _bugs\r\ngroup by 1,2\r\norder by 3 desc\r\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Bug Age by Epic", - "type": "barchart" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 135 - }, - "id": 78, - "options": { - "content": "
\n
\n \"No.4\"\n

Operation

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. The average incident age.\n2. Incident age refers to the length of time from incident creation to resolution.\n3. The incidents being calculated are filtered by \"incident resolution time\" (time filter at the upper-right corner) and \"Jira board\"(\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 139 - }, - "id": 64, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "/^value$/", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \r\n now() as time,\r\n avg(lead_time/1440) as value\r\nfrom jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\nwhere \r\n std_type = 'Incident'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\ngroup by 1;", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Incident Age (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The incident age at which 80% incidents‘ age lies below it.\n2. The incidents being calculated are filtered by \"incident resolution time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 139 - }, - "id": 55, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n ji.lead_time,\r\n percent_rank() over (order by lead_time asc) as ranks\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Incident'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect\r\n now() as time,\r\n max(lead_time/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8\r\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "80th Percentile Incident Age (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The average and 80th percentile incident age over time.\n2. The time granularity can be switched to Week or Month by \"Time Interval\" above. \n3. When Time Interval is set to \"Month\", the average_incident_age of \"2021-06-01\" refers to the average age of incidents whose resolution time falls under [2020-06-01, 2020-07-01).", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Incident Age (day)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 139 - }, - "id": 57, - "interval": "", - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _incidents as(\r\n select \r\n ji.issue_id,\r\n DATE_ADD(date(resolution_date), INTERVAL -$interval(date(resolution_date))+1 DAY) as time,\r\n lead_time,\r\n round(lead_time/1440) as lead_time_day\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Incident'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n timestamp(time) as time,\r\n avg(lead_time/1440) as \"Average Lead Time\"\r\nfrom _incidents\r\ngroup by 1\r\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Incident Age over Time", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. The cumulative distribution of incident age.\n2. Each point refers to the percent rank of a incident age.", - "fieldConfig": { - "defaults": { - "unit": "percentunit" - }, - "overrides": [] - }, - "fill": 0, - "fillGradient": 4, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 145 - }, - "hiddenSeries": false, - "id": 68, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 8, - "nullPointMode": "null", - "options": { - "alertThreshold": false - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 0.5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\r\n select \r\n round(ji.lead_time/1440) as lead_time_day\r\n from jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Incident'\r\n and std_status = 'Resolved'\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [ - { - "$$hashKey": "object:1070", - "colorMode": "ok", - "fill": true, - "line": true, - "op": "lt", - "value": 0.8, - "yaxis": "right" - } - ], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Incident Age (Cumulative Distribution)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:76", - "format": "percentunit", - "label": "Percent Rank (%)", - "logBase": 1, - "max": "1.2", - "min": null, - "show": true - }, - { - "$$hashKey": "object:77", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "mysql", - "description": "The average time spent on fixing incidents under different Jira Epics.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Incident Age(day)", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 151 - }, - "id": 67, - "options": { - "barWidth": 0.5, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": { - "valueSize": 1 - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _incidents as(\r\n select\r\n ji.issue_id,\r\n ji.epic_key,\r\n ji.lead_time\r\n from\r\n jira_issues ji\r\n join jira_board_issues jbi on ji.issue_id = jbi.issue_id\r\n where \r\n std_type = 'Incident'\r\n and std_status = 'Resolved'\r\n and epic_key is not null\r\n and epic_key != ''\r\n and $__timeFilter(resolution_date)\r\n and jbi.board_id = $board_id\r\n)\r\n\r\nselect \r\n now() as time,\r\n epic_key as \"Epic Key\",\r\n avg(lead_time)/1440 as \"Incident Age\"\r\nfrom _incidents\r\ngroup by 1,2\r\norder by 3 desc\r\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Incident Age by Epic", - "type": "barchart" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "velocity", - "user_value" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "Month", - "value": "DAY" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Time Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": false, - "text": "Week", - "value": "DAYOFWEEK" - }, - { - "selected": true, - "text": "Month", - "value": "DAY" - } - ], - "query": "Week : DAYOFWEEK, Month : DAY", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "mysql", - "definition": "select distinct concat(name, ': ', board_id) from jira_boards", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Choose Board", - "multi": true, - "name": "board_id", - "options": [], - "query": "select distinct concat(name, ': ', board_id) from jira_boards", - "refresh": 1, - "regex": "/^(?[^:]+): (?\\d+)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - }, - { - "allValue": null, - "current": { - "selected": true, - "text": [ - "All" - ], - "value": [ - "$__all" - ] - }, - "datasource": "mysql", - "definition": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Choose Repo", - "multi": true, - "name": "repo_id", - "options": [], - "query": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "refresh": 1, - "regex": "/^(?[^:]+): (?\\d+)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-5y", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Delivery Velocity (require Jira and Gitlab data)", - "uid": "ZfaFQeM7k", - "version": 20 -} diff --git a/grafana/_archive/GithubDomainLayerOnly.json b/grafana/_archive/GithubDomainLayerOnly.json deleted file mode 100644 index d15048c482a..00000000000 --- a/grafana/_archive/GithubDomainLayerOnly.json +++ /dev/null @@ -1,622 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 10, - "links": [], - "panels": [ - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 0 - }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n sum(additions)\nFROM commits\nWHERE\n $__timeFilter(created_at)\n AND message not like '%Merge branch%'", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Added LOC", - "type": "stat" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 0 - }, - "id": 12, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n sum(deletions)\nFROM commits\nWHERE\n $__timeFilter(created_at) \n AND message not like '%Merge branch%'\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Deleted Loc", - "type": "stat" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 13, - "w": 12, - "x": 8, - "y": 0 - }, - "id": 14, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with loc as(\n SELECT\n DATE_ADD(date(authored_date), INTERVAL 1 DAY) as time,\n sum(additions) as added_LOC,\n sum(deletions) as deleted_LOC\n FROM commits\n WHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n group by 1\n)\n\nSELECT \n timestamp(time) as time,\n added_LOC as \"Added LOC\",\n deleted_LOC as \"Deleted LOC\"\nFROM loc\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Added and Deleted LOC Over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 6 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n count(distinct author_email) as developer_count\nFROM commits", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total Developers By Email", - "type": "stat" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 6 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM \n prs\nWHERE\n $__timeFilter(created_at) AND state = 'open'\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Open Pull Requests", - "type": "stat" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 0, - "y": 13 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM \n prs\nWHERE\n $__timeFilter(created_at)\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total Pull Request Count", - "type": "stat" - }, - { - "datasource": "mysql", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 4, - "x": 4, - "y": 13 - }, - "id": 2, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n count(sha)\n \nFROM commits\nWHERE\n $__timeFilter(created_at)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "duration_sec" - ], - "type": "column" - } - ] - ], - "table": "builds", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "title": "Total Commits", - "type": "stat" - } - ], - "schemaVersion": 30, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "2021-01-30T18:24:57.517Z", - "to": "2022-02-02T18:24:57.519Z" - }, - "timepicker": {}, - "timezone": "", - "title": "Github Domain Layer Only", - "uid": "ilRLEaFnz", - "version": 5 -} diff --git a/grafana/_archive/Gitlab.json b/grafana/_archive/Gitlab.json deleted file mode 100644 index 1baf386b8c6..00000000000 --- a/grafana/_archive/Gitlab.json +++ /dev/null @@ -1,2186 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 10, - "iteration": 1638297448523, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": false, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "Data Source Specific Dashboard" - ], - "targetBlank": false, - "title": "Metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 48, - "options": { - "content": "
\n
\n \"No.1\"\n

MR Troughput and Pass Rate

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of Pull/Merge request created.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 4 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM \n gitlab_merge_requests gmr\nWHERE\n gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Merge Request Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Total number of Pull/Merge request merged.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 4 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n count(*) as value\nFROM\n gitlab_merge_requests gmr\nWHERE\n gmr.project_id = $repo_id\n and state = 'merged'\n and $__timeFilter(gitlab_created_at)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Merge Request Pass Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Pull Request Pass Count/Pull Request Count", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0.8 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 4 - }, - "id": 37, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n 1.0 * count(case when state = 'merged' then 1 else null end)/count(*)\nFROM\n gitlab_merge_requests gmr\nWHERE\n gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Merge Request Pass Rate (%)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Pull Request Pass Rate over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", value \"pull_request_pass_rate\" of \"2021-06-01\" calculates the PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 8, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _mr_request_rate as(\n SELECT\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n state\n FROM\n gitlab_merge_requests gmr\n WHERE\n gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\n)\n\nSELECT \n timestamp(time) as time,\n 1.0 * count(case when state = 'merged' then 1 else null end)/count(*) as \"Pull Request Pass Rate\"\nFROM _mr_request_rate\nGROUP BY 1\nORDER BY 1\n", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Merge Request Pass Rate over Time", - "type": "timeseries" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 16 - }, - "id": 50, - "options": { - "content": "
\n
\n \"No.2\"\n

MR Review Metrics

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Average PR/MR review time.\n2. PR/MR review time refers to the time between the creation time and merged time of a PR/MR.\n3. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 20 - }, - "id": 80, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \n now() as time,\n avg(TIMESTAMPDIFF(MINUTE,gitlab_created_at,merged_at))/1440 as value\nfrom gitlab_merge_requests\nWHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Avg Merge Request Review Time (day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The 80th percentile PR/MR review time.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 7 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 20 - }, - "id": 83, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _ranks as(\n select \n TIMESTAMPDIFF(SECOND,gitlab_created_at,merged_at)/86400 as metric,\n percent_rank() over (order by TIMESTAMPDIFF(MINUTE,gitlab_created_at,merged_at) asc) as ranks\n from gitlab_merge_requests\n WHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n )\n \nselect\n now() as time,\n max(metric) as value\nfrom _ranks\nwhere \n ranks <= 0.8\ngroup by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "80th Percentile Merge Request Review Time(day)", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. The average and 80th percentile PR/MR review time over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", the average review time of \"2021-06-01\" refers to the review time of PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Review Time (day)", - "axisPlacement": "auto", - "axisSoftMin": -3, - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 20 - }, - "id": 82, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with reviewed_mr as(\n select\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n TIMESTAMPDIFF(SECOND,first_comment_time,merged_at)/86400 as review_time\n from gitlab_merge_requests\n WHERE\n merged_at is not null\n and $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n)\n\nselect \n timestamp(time) as time,\n avg(review_time) as \"Average Review Time\"\nfrom reviewed_mr\ngroup by 1\norder by 1 asc", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Merge Request Review Time over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The average round of PR/MR review.", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 26 - }, - "id": 53, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n avg(review_rounds) as value\nFROM \n gitlab_merge_requests gmr\nWHERE\n state = 'merged'\n and review_rounds > 0\n and gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Merge Request Review Round", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Avg PR/MR Review Round over time.\n2. The time granularity can be switched to week or month by \"Time Interval\" above. \n3. When Time Interval is set to \"month\", value \"Avg Pull Request Review Round\" of \"2021-06-01\" calculates the PR/MR whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 26 - }, - "id": 55, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "SELECT\n timestamp(DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY)) as time,\n avg(review_rounds) as \"Pull Request Reveiw Round\"\nFROM\n gitlab_merge_requests gmr\nWHERE\n state = 'merged'\n and review_rounds > 0\n and gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\nGROUP BY 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Merge Request Review Round over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The average round of PR/MR review of PR authors.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 54, - "options": { - "barWidth": 0.52, - "groupWidth": 0.7, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n author_username as Author,\n avg(review_rounds) as value\nFROM \n gitlab_merge_requests gmr\nWHERE\n state = 'merged'\n and review_rounds > 0\n and gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\nGROUP BY 1,2\nORDER BY 3 desc\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Merge Request Review Round by Author", - "type": "barchart" - }, - { - "datasource": "mysql", - "description": "One-time pass rate for PR/MR over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "One-time Pass Rate(%)", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 38 - }, - "id": 56, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "queryType": "randomWalk", - "rawQuery": true, - "rawSql": "with _review_rounds as(\n SELECT\n DATE_ADD(date(gitlab_created_at), INTERVAL -$interval(date(gitlab_created_at))+1 DAY) as time,\n review_rounds\n FROM\n gitlab_merge_requests gmr\n WHERE\n state = 'merged'\n and review_rounds > 0\n and gmr.project_id = $repo_id\n and $__timeFilter(gitlab_created_at)\n)\n\nSELECT \n timestamp(time) as time,\n count(case when review_rounds = 1 then true else null end)/count(*) as one_time_pass_rate\nFROM \n _review_rounds\nGROUP BY 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "value" - ], - "type": "column" - } - ] - ], - "timeColumn": "time", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "One-time Pass Rate for Code Reviews over Time", - "type": "timeseries" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 44 - }, - "id": 52, - "options": { - "content": "
\n
\n \"No.3\"\n

Commit Metrics

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Total number of commits created.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "decimals": 1, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 48 - }, - "id": 34, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS \"time\",\n count(*) as value\nFROM gitlab_commits\nWHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of commits created over time.\n2. When Time Interval is set to \"month\", the commit_count\" of \"2021-06-01\" calculates the commits whose creation time falls under [2020-06-01, 2020-07-01)", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "Commit Count", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 6, - "y": 48 - }, - "id": 36, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _commits as(\n SELECT\n DATE_ADD(date(authored_date), INTERVAL -$interval(date(authored_date))+1 DAY) as time,\n count(*) as commit_count\n FROM gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\n group by 1\n)\n\nSELECT \n timestamp(time) as time,\n commit_count as \"Commit Count\"\nFROM _commits\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Count over Time", - "type": "timeseries" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "The number of commits from different repos.", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 48 - }, - "hiddenSeries": false, - "id": 87, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n gp.name as metric,\n count(*) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id\nWHERE\n $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1,2\norder by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Commit Count by Repo", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "series", - "name": null, - "show": true, - "values": [ - "total" - ] - }, - "yaxes": [ - { - "$$hashKey": "object:89", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:90", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "mysql", - "description": "1. Sum of the number of added lines of code under all commits.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 54 - }, - "id": 42, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n sum(additions) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id \nWHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added LOC", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Sum of the number of deleted lines of code under all commits.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner) and \"Gitlab repo\"(\"Choose Repo\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 54 - }, - "id": 43, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n sum(deletions) as value\nFROM gitlab_commits gc\n join gitlab_projects gp on gc.project_id = gp.gitlab_id \nWHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Deleted LOC", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Added/deleted number of lines of code over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "LOC", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 54 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with loc as(\n SELECT\n DATE_ADD(date(authored_date), INTERVAL -$interval(date(authored_date))+1 DAY) as time,\n sum(additions) as added_LOC,\n sum(deletions) as deleted_LOC\n FROM gitlab_commits\n WHERE\n message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and project_id = $repo_id\n group by 1\n)\n\nSELECT \n timestamp(time) as time,\n added_LOC as \"Added LOC\",\n deleted_LOC as \"Deleted LOC\"\nFROM loc\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added and Deleted LOC over Time", - "type": "timeseries" - }, - { - "datasource": "mysql", - "description": "The number of added/deleted LOC from different commit authors.", - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "orange", - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "LOC", - "axisPlacement": "auto", - "axisSoftMin": 0, - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineWidth": 1 - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 60 - }, - "id": 49, - "options": { - "barWidth": 0.9, - "groupWidth": 0.6, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "orientation": "auto", - "showValue": "auto", - "text": {}, - "tooltip": { - "mode": "multi" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() as time,\n sum(additions) as \"Added LOC\",\n sum(deletions) as \"Removed LOC\",\n gc.author_name\nFROM gitlab_commits gc \n join gitlab_projects gp on gc.project_id = gp.gitlab_id\nWHERE\n author_name is not null\n and message not like '%Merge branch%'\n and $__timeFilter(authored_date)\n and gc.project_id = $repo_id\ngroup by 1,4\norder by 2 desc\nlimit 20", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Added and Deleted LOC by Author", - "type": "barchart" - }, - { - "datasource": null, - "gridPos": { - "h": 4, - "w": 24, - "x": 0, - "y": 66 - }, - "id": 97, - "options": { - "content": "
\n
\n \"No.4\"\n

Development Cost Metrics

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "queryType": "randomWalk", - "refId": "A" - } - ], - "transparent": true, - "type": "text" - }, - { - "datasource": "mysql", - "description": "1. Number of people who have created or reviewed a Pull/Merge Request.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 90, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with all_developers as(\n select \n gitlab_reviewers.name as user_name\n from gitlab_merge_requests\n join gitlab_reviewers on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\n\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_merge_requests.project_id = $repo_id\n union\n select \n distinct author_username as user_name \n from gitlab_merge_requests\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_merge_requests.project_id = $repo_id\n union\n select \n distinct author_name as user_name \n from gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and gitlab_commits.project_id = $repo_id\n)\n\n\nSELECT\n now() AS \"time\",\n count(distinct user_name) as developer_count\nFROM all_developers", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total Developer Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Total number of Commit Authors.\n2. The commits being calculated are filtered by \"authored_date\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 76 - }, - "id": 92, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n count(distinct author_name) as value\nFROM gitlab_commits\nWHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\ngroup by 1\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Commit Author Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "1. Number of Pull/Merge Request Reviewers.\n2. The PR/MR being calculated are filtered by \"PR/MR creation time\" (time filter at the upper-right corner).", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 8, - "y": 76 - }, - "id": 94, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "select \n now() as time,\n count(distinct name)\nfrom\n gitlab_reviewers", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Reviewer Count", - "type": "stat" - }, - { - "datasource": "mysql", - "description": "Pull Request Reviewer Count/Total Developer Count", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0.2 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 16, - "y": 76 - }, - "id": 96, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with all_user_names as(\n select \n distinct gitlab_reviewers.name as user_name\n from gitlab_reviewers\n join gitlab_merge_requests on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\n WHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_reviewers.project_id = $repo_id\n union\n select \n distinct author_username as user_name \n from gitlab_merge_requests\n WHERE\n $__timeFilter(gitlab_created_at)\n and project_id = $repo_id\n union\n select \n distinct author_name as user_name \n from gitlab_commits\n WHERE\n $__timeFilter(authored_date)\n and project_id = $repo_id\n),\n\nreviewer as (\n select distinct gitlab_reviewers.name from gitlab_reviewers\n join gitlab_merge_requests on gitlab_reviewers.merge_request_id = gitlab_merge_requests.gitlab_id\nWHERE\n $__timeFilter(gitlab_merge_requests.gitlab_created_at)\n and gitlab_reviewers.project_id = $repo_id\n)\n\nSELECT\n now() AS \"time\",\n 1.0*count(distinct reviewer.name)/count(distinct user_name) as value\nFROM reviewer, all_user_names", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Pull Request Reviewer Rate (%)", - "type": "stat" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "Gitlab", - "Data Source Specific Dashboard" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": false, - "text": "Month", - "value": "DAY" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Time Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": false, - "text": "Week", - "value": "DAYOFWEEK" - }, - { - "selected": true, - "text": "Month", - "value": "DAY" - } - ], - "query": "Week : DAYOFWEEK, Month : DAY", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - }, - { - "allValue": null, - "current": { - "selected": false, - "text": "All", - "value": "$__all" - }, - "datasource": "mysql", - "definition": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "description": null, - "error": null, - "hide": 0, - "includeAll": true, - "label": "Choose Repo", - "multi": false, - "name": "repo_id", - "options": [], - "query": "select distinct concat(name, ': ', gitlab_id) from gitlab_projects", - "refresh": 1, - "regex": "/^(?[^:]+): (?\\d+)$/", - "skipUrlSync": false, - "sort": 0, - "type": "query" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Gitlab", - "uid": "6YcsxLN7z", - "version": 2 -} diff --git a/grafana/_archive/Jenkins.json b/grafana/_archive/Jenkins.json deleted file mode 100644 index 049b88e5a6a..00000000000 --- a/grafana/_archive/Jenkins.json +++ /dev/null @@ -1,470 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 5, - "iteration": 1637051361097, - "links": [ - { - "asDropdown": false, - "icon": "bolt", - "includeVars": false, - "keepTime": true, - "tags": [], - "targetBlank": false, - "title": "Homepage", - "tooltip": "", - "type": "link", - "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" - }, - { - "asDropdown": false, - "icon": "external link", - "includeVars": false, - "keepTime": true, - "tags": [ - "Data Source Specific Dashboard" - ], - "targetBlank": false, - "title": "metric dashboards", - "tooltip": "", - "type": "dashboards", - "url": "" - } - ], - "panels": [ - { - "datasource": "mysql", - "description": "1. Number of builds executed.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 0 - }, - "id": 4, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n count(*)\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Count", - "type": "stat" - }, - { - "cacheTimeout": null, - "datasource": "mysql", - "description": "The percentage of successful, failed, and aborted builds.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "decimals": 0, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 0 - }, - "id": 37, - "interval": null, - "links": [], - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "right", - "values": [ - "percent", - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": true - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "table", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n result as metric,\n count(*) as value\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nGROUP BY 1,2\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Result Distribution", - "type": "piechart" - }, - { - "datasource": "mysql", - "description": "Number of successful builds / Number of total builds", - "fieldConfig": { - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 6 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "text": {}, - "textMode": "auto" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "SELECT\n now() AS time,\n 1.0 * count(case when result = 'SUCCESS' then 1 else null end)/count(*)\nFROM jenkins_builds\nWHERE\n $__timeFilter(start_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "project_id" - ], - "type": "column" - } - ] - ], - "table": "gitlab_commits", - "timeColumn": "created_at", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Build Success Rate", - "type": "stat" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "mysql", - "description": "1. Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", - "fieldConfig": { - "defaults": { - "unit": "percentunit" - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 6 - }, - "hiddenSeries": false, - "id": 50, - "interval": "", - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.0.6", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": true, - "rawSql": "with _build_success_rate as(\r\n SELECT\r\n DATE_ADD(date(start_time), INTERVAL -$interval(date(start_time))+1 DAY) as time,\r\n result\r\n FROM\r\n jenkins_builds\r\n WHERE\r\n $__timeFilter(start_time)\r\n)\r\n\r\nSELECT \r\n timestamp(time) as time,\r\n 1.0 * sum(case when result = 'SUCCESS' then 1 else 0 end)/ count(*) as \"Build Success Rate\"\r\nFROM _build_success_rate\r\ngroup by 1\r\norder by 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Build Success Rate over Time", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:262", - "decimals": null, - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:263", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [ - "Jenkins", - "Data Source Specific Dashboard" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "selected": true, - "text": "Month", - "value": "DAY" - }, - "description": null, - "error": null, - "hide": 0, - "includeAll": false, - "label": "Time Interval", - "multi": false, - "name": "interval", - "options": [ - { - "selected": false, - "text": "Week", - "value": "DAYOFWEEK" - }, - { - "selected": true, - "text": "Month", - "value": "DAY" - } - ], - "query": "Week : DAYOFWEEK, Month : DAY", - "queryValue": "", - "skipUrlSync": false, - "type": "custom" - } - ] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Jenkins", - "uid": "zDsUxYHnz", - "version": 13894 -} diff --git a/grafana/_archive/UserValueSpecificDashboardsHomepage.json b/grafana/_archive/UserValueSpecificDashboardsHomepage.json deleted file mode 100644 index 4c1be1f9b2d..00000000000 --- a/grafana/_archive/UserValueSpecificDashboardsHomepage.json +++ /dev/null @@ -1,344 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 4, - "links": [], - "panels": [ - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 0, - "y": 0 - }, - "id": 10, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"velocity\"\n

Delivery Velocity

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 7, - "y": 0 - }, - "id": 5, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"quality\"\n

Delivery Quality

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "datasource": null, - "gridPos": { - "h": 18, - "w": 10, - "x": 14, - "y": 0 - }, - "id": 9, - "links": [], - "options": { - "content": "![Merico Logo](/public/img/lake/logo.png \"Merico\")\n***\n# **MARI Introduction**\n***\nMARI is a methodology developed specifically to bring the process of data driven software engineering to organizations of all sizes in a simple and organized way; empowering them to translate measurement into insight, and insight into results. MARI is a cycle of iterations, focused on mapping challenges to metrics, metrics to actions, and actions to improvements. \n

\n\n\n#### *M (Measure)*\n \nGreat answers require great questions. To get started, identify specific and clearly definable pain points in your development processes or teams, set tightly constrained goals, and then with team consensus, prioritize those goals. From here, identify 'North Star' metrics and measurements that map to each goal. Once you have clearly defined and documented the pain-points, mapped them to metrics, and prioritized, it's time to standardize these metrics and methods across teams. \n\n##### *A (Analyze)*\n\nEffective measures are only as good as the analysis that gets applied to them. To understand productivity in an effective and quantitative way, look towards key attributes like trend, distribution, variability, dispersion, and periodicity. These analysis techniques can uncover problems and opportunities. \n\n##### *R (Review)*\n\nOnce you have equipped yourself with the results of the analysis, it's time to coherently review them and discover what hinders your team from reaching full potential. With key stakeholders in the conversation, assess your workflow, toolset, developer satisfaction and many other aspects with an eye towards identifying cause and effect. It is critical that this part of the process is focused on the real root causes, not blame. Review should involve those affected, and those with the responsibility. \n\n##### *I (Improve)*\n\nAfter review, it's time to prepare a defined and clear iteration. What actions can be taken to potentially address the areas of opportunity? These actions should be clear, defined, measurable, and focused enough that success or failure can be clearly identified and attributed. In larger organizations, consider A/B testing the action plan between a few teams, and be sure to test and assess action plans at smaller scale before implementing enterprise-wide. \n \nAs you implement action plans for improvement, refresh your MARI cycle to coherently drive progress and evolution.\n\n", - "mode": "markdown" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 0, - "y": 9 - }, - "id": 7, - "interval": null, - "links": [], - "options": { - "content": " \n
\n \"cost\"\n

Delivery Cost

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "gridPos": { - "h": 9, - "w": 7, - "x": 7, - "y": 9 - }, - "id": 13, - "interval": null, - "links": [], - "options": { - "content": " \n
\n \"capability\"\n

Delivery Capability

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "type": "text" - }, - { - "cacheTimeout": null, - "datasource": null, - "description": "", - "gridPos": { - "h": 9, - "w": 7, - "x": 0, - "y": 18 - }, - "id": 12, - "interval": null, - "links": [], - "options": { - "content": "\n
\n \"requirement\"\n

New Dashboard

\n
\n
", - "mode": "html" - }, - "pluginVersion": "8.0.6", - "targets": [ - { - "format": "time_series", - "group": [], - "metricColumn": "none", - "rawQuery": false, - "rawSql": "SELECT\n create_time AS \"time\",\n progress\nFROM ca_analysis\nWHERE\n $__timeFilter(create_time)\nORDER BY 1", - "refId": "A", - "select": [ - [ - { - "params": [ - "progress" - ], - "type": "column" - } - ] - ], - "table": "ca_analysis", - "timeColumn": "create_time", - "timeColumnType": "timestamp", - "where": [ - { - "name": "$__timeFilter", - "params": [], - "type": "macro" - } - ] - } - ], - "timeFrom": null, - "timeShift": null, - "transparent": true, - "type": "text" - } - ], - "refresh": "", - "schemaVersion": 30, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-6M", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "User Value Specific Dashboards-Homepage", - "uid": "RXJZNpMnz", - "version": 14 -} diff --git a/grafana/dashboards/ClaudeCodeAdoption.json b/grafana/dashboards/ClaudeCodeAdoption.json new file mode 100644 index 00000000000..5d08946f923 --- /dev/null +++ b/grafana/dashboards/ClaudeCodeAdoption.json @@ -0,0 +1,1295 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Latest daily active user count from the organization summary feed", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT daily_active_user_count as \"Daily Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Daily Active Users", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Latest weekly active user count from the organization summary feed", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT weekly_active_user_count as \"Weekly Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Weekly Active Users", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Latest monthly active user count from the organization summary feed", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT monthly_active_user_count as \"Monthly Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Monthly Active Users", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Latest assigned seat count from the organization summary feed", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT assigned_seat_count as \"Assigned Seats\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Assigned Seats", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Latest pending invite count from the organization summary feed", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT pending_invite_count as \"Pending Invites\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Pending Invites", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distinct Claude Code users with code activity on the most recent available analytics date", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) as \"Active Claude Code Users\"\nFROM _tool_claude_code_user_activity\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND `date` = (\n SELECT MAX(`date`)\n FROM _tool_claude_code_user_activity\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n )\n AND (\n cc_session_count > 0\n OR cc_commit_count > 0\n OR cc_pull_request_count > 0\n OR cc_lines_added > 0\n OR cc_lines_removed > 0\n OR edit_tool_accepted + edit_tool_rejected + multi_edit_tool_accepted + multi_edit_tool_rejected + write_tool_accepted + write_tool_rejected + notebook_edit_tool_accepted + notebook_edit_tool_rejected > 0\n )", + "refId": "A" + } + ], + "title": "Active Claude Code Users", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total Claude Code sessions in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(cc_session_count) as \"Sessions\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Sessions", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Accepted tool actions as a percentage of all accepted and rejected tool actions", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 40 + }, + { + "color": "green", + "value": 70 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(\n SUM(edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted) * 100.0\n / NULLIF(\n SUM(\n edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted\n + edit_tool_rejected + multi_edit_tool_rejected + write_tool_rejected + notebook_edit_tool_rejected\n ),\n 0\n ),\n 1\n) as \"Acceptance Rate %\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Tool Acceptance Rate", + "type": "gauge" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total lines added by Claude Code across the organization in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(cc_lines_added) as \"Lines Added\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Lines Added", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total commits attributed to Claude Code in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(cc_commit_count) as \"Commits\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Commits by Claude Code", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total pull requests attributed to Claude Code in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 8 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(cc_pull_request_count) as \"PRs\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "PRs by Claude Code", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "How many days have elapsed since the newest row across the migrated Claude analytics tables; Anthropic analytics normally lag by about 3 days", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 8 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(TIMESTAMPDIFF(DAY, MAX(latest_date), UTC_DATE()), 9999) as \"Days Since Latest Data\"\nFROM (\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_user_activity\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_activity_summary\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_chat_project\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_skill_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_connector_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n) latest_dates", + "refId": "A" + } + ], + "title": "Data Freshness", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily, weekly, and monthly active user counts over time from the organization summary feed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n daily_active_user_count as \"Daily Active Users\",\n weekly_active_user_count as \"Weekly Active Users\",\n monthly_active_user_count as \"Monthly Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY 1", + "refId": "A" + } + ], + "title": "Org Active Users Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily Claude Code sessions and distinct users with code activity over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n SUM(cc_session_count) as \"Sessions\",\n COUNT(DISTINCT CASE\n WHEN (\n cc_session_count > 0\n OR cc_commit_count > 0\n OR cc_pull_request_count > 0\n OR cc_lines_added > 0\n OR cc_lines_removed > 0\n OR edit_tool_accepted + edit_tool_rejected + multi_edit_tool_accepted + multi_edit_tool_rejected + write_tool_accepted + write_tool_rejected + notebook_edit_tool_accepted + notebook_edit_tool_rejected > 0\n ) THEN user_id\n ELSE NULL\n END) as \"Active Users\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY `date`\nORDER BY 1", + "refId": "A" + } + ], + "title": "Sessions and Active Claude Code Users Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Lines added and removed over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n SUM(cc_lines_added) as \"Lines Added\",\n SUM(cc_lines_removed) as \"Lines Removed\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY `date`\nORDER BY 1", + "refId": "A" + } + ], + "title": "Code Churn Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Assigned seats and pending invites over time from the organization summary feed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n assigned_seat_count as \"Assigned Seats\",\n pending_invite_count as \"Pending Invites\"\nFROM _tool_claude_code_activity_summary\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY 1", + "refId": "A" + } + ], + "title": "Seats and Pending Invites Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Accepted and rejected tool actions, grouped by Claude Code tool", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT tool_name as \"Tool\",\n SUM(accepted) as \"Accepted\",\n SUM(rejected) as \"Rejected\",\n ROUND(SUM(accepted) * 100.0 / NULLIF(SUM(accepted + rejected), 0), 1) as \"Acceptance Rate %\"\nFROM (\n SELECT `date`, connection_id, scope_id, 'edit_tool' as tool_name, edit_tool_accepted as accepted, edit_tool_rejected as rejected\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'multi_edit_tool' as tool_name, multi_edit_tool_accepted as accepted, multi_edit_tool_rejected as rejected\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'write_tool' as tool_name, write_tool_accepted as accepted, write_tool_rejected as rejected\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'notebook_edit_tool' as tool_name, notebook_edit_tool_accepted as accepted, notebook_edit_tool_rejected as rejected\n FROM _tool_claude_code_user_activity\n) tool_actions\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY tool_name\nORDER BY SUM(accepted + rejected) DESC, tool_name", + "refId": "A" + } + ], + "title": "Tool Acceptance by Tool", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Peak daily users and session counts grouped by Claude skill usage", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT skill_name as \"Skill\",\n MAX(distinct_user_count) as \"Peak Daily Users\",\n SUM(chat_conversation_count) as \"Chat Conversations\",\n SUM(cc_session_count) as \"Claude Code Sessions\"\nFROM _tool_claude_code_skill_usage\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY skill_name\nORDER BY SUM(cc_session_count + chat_conversation_count) DESC, skill_name", + "refId": "A" + } + ], + "title": "Skill Usage by Skill", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Peak daily users and session counts grouped by connector usage", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT connector_name as \"Connector\",\n MAX(distinct_user_count) as \"Peak Daily Users\",\n SUM(chat_conversation_count) as \"Chat Conversations\",\n SUM(cc_session_count) as \"Claude Code Sessions\"\nFROM _tool_claude_code_connector_usage\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY connector_name\nORDER BY SUM(cc_session_count + chat_conversation_count) DESC, connector_name", + "refId": "A" + } + ], + "title": "Connector Usage by Connector", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Project-level chat adoption with only project aggregates; user identifiers are intentionally excluded from this org dashboard", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 20, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(project_name, ''), project_id) as \"Project\",\n MAX(distinct_user_count) as \"Peak Daily Users\",\n SUM(conversation_count) as \"Conversations\",\n SUM(message_count) as \"Messages\"\nFROM _tool_claude_code_chat_project\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nGROUP BY COALESCE(NULLIF(project_name, ''), project_id)\nORDER BY SUM(message_count + conversation_count) DESC, 1\nLIMIT 25", + "refId": "A" + } + ], + "title": "Chat Project Usage", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Diagnostics table for row counts and freshest available date per Claude metric table", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT *\nFROM (\n SELECT '_tool_claude_code_activity_summary' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_activity_summary\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_user_activity' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_user_activity\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_skill_usage' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_skill_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_connector_usage' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_connector_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_chat_project' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_chat_project\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n) data_volume\nORDER BY row_count DESC, table_name", + "refId": "A" + } + ], + "title": "Data Volume by Table", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "claude-code", + "devlake", + "org" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_claude_code_scopes ORDER BY connection_id DESC", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_claude_code_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT id as scope_id FROM _tool_claude_code_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_claude_code_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Claude Code Adoption Overview", + "uid": "claude_code_adoption", + "version": 3, + "weekStart": "" +} diff --git a/grafana/dashboards/ClaudeCodeAdoptionByUser.json b/grafana/dashboards/ClaudeCodeAdoptionByUser.json new file mode 100644 index 00000000000..85b1335a6dc --- /dev/null +++ b/grafana/dashboards/ClaudeCodeAdoptionByUser.json @@ -0,0 +1,1301 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Org-level daily active user count; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT daily_active_user_count as \"Org Daily Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Org Daily Active Users", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Org-level assigned seat count; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT assigned_seat_count as \"Org Assigned Seats\"\nFROM _tool_claude_code_activity_summary\nWHERE connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY `date` DESC\nLIMIT 1", + "refId": "A" + } + ], + "title": "Org Assigned Seats", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Org-level session total for the selected time range; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_session_count), 0) as \"Org Sessions\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Org Sessions", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Days since the freshest Claude analytics row across the registered metric tables; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 4 + }, + { + "color": "red", + "value": 8 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(TIMESTAMPDIFF(DAY, MAX(latest_date), UTC_DATE()), 9999) as \"Days Since Latest Data\"\nFROM (\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_user_activity\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_activity_summary\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_chat_project\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_skill_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT MAX(`date`) as latest_date\n FROM _tool_claude_code_connector_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n) latest_dates", + "refId": "A" + } + ], + "title": "Data Freshness", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 5, + "panels": [], + "title": "Selected User Metrics", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Sessions for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_session_count), 0) as \"Selected User Sessions\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Sessions", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Tool acceptance rate for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 40 + }, + { + "color": "green", + "value": 70 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 7, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(\n SUM(edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted) * 100.0\n / NULLIF(\n SUM(\n edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted\n + edit_tool_rejected + multi_edit_tool_rejected + write_tool_rejected + notebook_edit_tool_rejected\n ),\n 0\n ),\n 1\n) as \"Selected User Acceptance Rate %\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Tool Acceptance Rate", + "type": "gauge" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Lines added for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_lines_added), 0) as \"Selected User Lines Added\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Lines Added", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Lines removed for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_lines_removed), 0) as \"Selected User Lines Removed\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Lines Removed", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Commits for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 9 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_commit_count), 0) as \"Selected User Commits\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Commits", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Pull requests for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 9 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(cc_pull_request_count), 0) as \"Selected User PRs\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User PRs", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Days with recorded Claude activity for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT CASE\n WHEN (\n cc_session_count > 0\n OR cc_commit_count > 0\n OR cc_pull_request_count > 0\n OR cc_lines_added > 0\n OR cc_lines_removed > 0\n OR chat_conversation_count > 0\n OR web_search_count > 0\n OR edit_tool_accepted + edit_tool_rejected + multi_edit_tool_accepted + multi_edit_tool_rejected + write_tool_accepted + write_tool_rejected + notebook_edit_tool_accepted + notebook_edit_tool_rejected > 0\n ) THEN `date`\n ELSE NULL\n END) as \"Selected User Active Days\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Active Days", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Chat conversations for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(SUM(chat_conversation_count), 0) as \"Selected User Chat Conversations\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )", + "refId": "A" + } + ], + "title": "Selected User Chat Conversations", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 14, + "panels": [], + "title": "Selected User Trends", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Sessions, commits, pull requests, and chat conversations over time for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n SUM(cc_session_count) as \"Sessions\",\n SUM(cc_commit_count) as \"Commits\",\n SUM(cc_pull_request_count) as \"PRs\",\n SUM(chat_conversation_count) as \"Chat Conversations\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )\nGROUP BY `date`\nORDER BY 1", + "refId": "A" + } + ], + "title": "Selected User Activity Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Code churn over time for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n SUM(cc_lines_added) as \"Lines Added\",\n SUM(cc_lines_removed) as \"Lines Removed\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )\nGROUP BY `date`\nORDER BY 1", + "refId": "A" + } + ], + "title": "Selected User Code Churn Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 17, + "panels": [], + "title": "Selected User Breakdowns", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Accepted and rejected tool actions for the selected user; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT tool_name as \"Tool\",\n SUM(accepted) as \"Accepted\",\n SUM(rejected) as \"Rejected\",\n ROUND(SUM(accepted) * 100.0 / NULLIF(SUM(accepted + rejected), 0), 1) as \"Acceptance Rate %\"\nFROM (\n SELECT `date`, connection_id, scope_id, 'edit_tool' as tool_name, edit_tool_accepted as accepted, edit_tool_rejected as rejected, user_id, user_email\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'multi_edit_tool' as tool_name, multi_edit_tool_accepted as accepted, multi_edit_tool_rejected as rejected, user_id, user_email\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'write_tool' as tool_name, write_tool_accepted as accepted, write_tool_rejected as rejected, user_id, user_email\n FROM _tool_claude_code_user_activity\n UNION ALL\n SELECT `date`, connection_id, scope_id, 'notebook_edit_tool' as tool_name, notebook_edit_tool_accepted as accepted, notebook_edit_tool_rejected as rejected, user_id, user_email\n FROM _tool_claude_code_user_activity\n) tool_actions\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )\nGROUP BY tool_name\nORDER BY SUM(accepted + rejected) DESC, tool_name", + "refId": "A" + } + ], + "title": "Selected User Tool Acceptance by Tool", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily activity details for the selected user without exposing user identity columns; admins and managers can inspect any user, other viewers are restricted to their own email-matched data", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 19, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as \"Day\",\n SUM(cc_session_count) as \"Sessions\",\n SUM(chat_conversation_count) as \"Chat Conversations\",\n SUM(web_search_count) as \"Web Searches\",\n SUM(cc_lines_added) as \"Lines Added\",\n SUM(cc_lines_removed) as \"Lines Removed\",\n SUM(cc_commit_count) as \"Commits\",\n SUM(cc_pull_request_count) as \"PRs\",\n ROUND(\n SUM(edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted) * 100.0\n / NULLIF(\n SUM(\n edit_tool_accepted + multi_edit_tool_accepted + write_tool_accepted + notebook_edit_tool_accepted\n + edit_tool_rejected + multi_edit_tool_rejected + write_tool_rejected + notebook_edit_tool_rejected\n ),\n 0\n ),\n 1\n ) as \"Tool Acceptance Rate %\"\nFROM _tool_claude_code_user_activity\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\n AND (\n ixd_get_user_role('${__user.email}') IN ('admin', 'manager')\n OR user_email = '${__user.email}'\n )\n AND (\n '${user:csv}' = '0'\n OR COALESCE(NULLIF(user_email, ''), user_id) = ${user:sqlstring}\n )\nGROUP BY `date`\nORDER BY `date` DESC\nLIMIT 60", + "refId": "A" + } + ], + "title": "Selected User Daily Activity", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 20, + "panels": [], + "title": "Org Context", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Org-level active user counts over time; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n daily_active_user_count as \"Daily Active Users\",\n weekly_active_user_count as \"Weekly Active Users\",\n monthly_active_user_count as \"Monthly Active Users\"\nFROM _tool_claude_code_activity_summary\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY 1", + "refId": "A" + } + ], + "title": "Org Active Users Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Org-level assigned seats and pending invites over time; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT `date` as time,\n assigned_seat_count as \"Assigned Seats\",\n pending_invite_count as \"Pending Invites\"\nFROM _tool_claude_code_activity_summary\nWHERE $__timeFilter(`date`)\n AND connection_id = ${connection_id}\n AND scope_id = '${scope_id}'\nORDER BY 1", + "refId": "A" + } + ], + "title": "Org Seats and Pending Invites Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 23, + "panels": [], + "title": "Diagnostics", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Diagnostics table for row counts and freshest available date per Claude metric table; intentionally ignores the Selected User filter", + "fieldConfig": { + "defaults": { + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 24, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT *\nFROM (\n SELECT '_tool_claude_code_activity_summary' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_activity_summary\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_user_activity' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_user_activity\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_skill_usage' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_skill_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_connector_usage' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_connector_usage\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_claude_code_chat_project' as table_name,\n COUNT(*) as row_count,\n MAX(`date`) as latest_date,\n COALESCE(TIMESTAMPDIFF(DAY, MAX(`date`), UTC_DATE()), 9999) as days_lag\n FROM _tool_claude_code_chat_project\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n) data_volume\nORDER BY row_count DESC, table_name", + "refId": "A" + } + ], + "title": "Data Volume by Table", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "claude-code", + "devlake", + "user", + "org" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_claude_code_scopes ORDER BY connection_id DESC", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_claude_code_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT id as scope_id FROM _tool_claude_code_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_claude_code_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "0", + "current": { + "selected": true, + "text": "All Accessible Users", + "value": "0" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT user_key as __value, user_key as __text FROM (SELECT COALESCE(NULLIF(user_email, ''), user_id) as user_key, user_email FROM _tool_claude_code_user_activity WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}') accessible_users WHERE ixd_get_user_role('${__user.email}') IN ('admin', 'manager') OR user_email = '${__user.email}' ORDER BY user_key", + "hide": 0, + "includeAll": true, + "label": "Selected User", + "multi": false, + "name": "user", + "options": [], + "query": "SELECT DISTINCT user_key as __value, user_key as __text FROM (SELECT COALESCE(NULLIF(user_email, ''), user_id) as user_key, user_email FROM _tool_claude_code_user_activity WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}') accessible_users WHERE ixd_get_user_role('${__user.email}') IN ('admin', 'manager') OR user_email = '${__user.email}' ORDER BY user_key", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Claude Code Adoption by User", + "uid": "claude_code_adoption_by_user", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/Linear.json b/grafana/dashboards/Linear.json new file mode 100644 index 00000000000..242e545b429 --- /dev/null +++ b/grafana/dashboards/Linear.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Linear", + "url": "https://devlake.apache.org/docs/Configuration/Linear" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Linear.\n- Data Source Required: Linear", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select concat(name, '--', id) from boards where id like 'linear%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'linear%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Linear", + "uid": "linear-dashboard", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/Rootly.json b/grafana/dashboards/Rootly.json new file mode 100644 index 00000000000..8cb182b2a7e --- /dev/null +++ b/grafana/dashboards/Rootly.json @@ -0,0 +1,1295 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 20, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Rootly", + "url": "https://devlake.apache.org/docs/Configuration/Rootly" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the incident data from Rootly.\n- Data Source Required: Rootly", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Incident Resolution Status", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.original_status = 'resolved'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Resolved Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 131, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n b.name as service,\r\n i.issue_key,\r\n i.description,\r\n i.original_status,\r\n i.priority,\r\n i.created_date,\r\n i.updated_date,\r\n round((i.lead_time_minutes/1440),1) as lead_time_days,\r\n i.url\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n join boards b on bi.board_id = b.id\r\nwhere \r\n $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of Incidents [Created in Selected Time Range]", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.original_status = 'resolved' then i.id else null end) as resolved_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * resolved_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate [Incidents created in the selected time range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Resolution Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.original_status = 'resolved' then i.id else null end)/count(distinct i.id) as resolved_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n resolved_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate over Time [Incidents Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Mean Time to Resolve (MTTR)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.original_status = 'resolved'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "MTTR [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.original_status = 'resolved'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Incidents' MTTR are less than # [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Incident Age(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_incident_age\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.original_status = 'resolved'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_incident_age\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean MTTR [Incidents Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "1. The cumulative distribution of MTTR\n2. Each point refers to the percent rank of a distinct duration to resolve incidents.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.original_status = 'resolved'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of MTTR [Incidents Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select concat(name, '--', id) from boards where id like 'rootly%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'rootly%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Rootly", + "uid": "rootly-dashboard", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/mysql/Asana.json b/grafana/dashboards/mysql/Asana.json new file mode 100644 index 00000000000..9021540d4e7 --- /dev/null +++ b/grafana/dashboards/mysql/Asana.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Asana", + "url": "https://devlake.apache.org/docs/Configuration/Asana" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Asana.\n- Data Source Required: Asana", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select \r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Issues\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Delivered Issues\"\r\nFROM issues i\r\n\tjoin board_issues bi on i.id = bi.issue_id\r\n\tjoin boards b on bi.board_id = b.id\r\nwhere \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect \r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select \r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere \r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select \r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect \r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select \r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where \r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect \r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select concat(name, '--', id) from boards where id like 'asana%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'asana%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Asana", + "uid": "asana-dashboard", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/Bamboo.json b/grafana/dashboards/mysql/Bamboo.json similarity index 95% rename from grafana/dashboards/Bamboo.json rename to grafana/dashboards/mysql/Bamboo.json index aa09186d94d..3269c059ec1 100644 --- a/grafana/dashboards/Bamboo.json +++ b/grafana/dashboards/mysql/Bamboo.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from Bamboo, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate). A build in Bamboo is an execution of a Bamboo Plan.\n- Data Source Required: Bamboo", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -65,7 +65,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of builds executed in the selected time range", "fieldConfig": { "defaults": { @@ -111,10 +111,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -165,7 +165,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful builds / Number of total builds", "fieldConfig": { "defaults": { @@ -214,10 +214,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -268,7 +268,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -401,10 +401,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -455,7 +455,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful builds / Number of total builds", "fieldConfig": { "defaults": { @@ -502,10 +502,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -556,7 +556,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -639,10 +639,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -694,7 +694,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -797,10 +797,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -851,7 +851,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -964,10 +964,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -1001,7 +1001,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1100,10 +1100,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1157,7 +1157,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1189,7 +1189,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from cicd_scopes where id like \"bamboo%\" ", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/DORA.json b/grafana/dashboards/mysql/DORA.json similarity index 97% rename from grafana/dashboards/DORA.json rename to grafana/dashboards/mysql/DORA.json index f820373d220..18e45f6b8e5 100644 --- a/grafana/dashboards/DORA.json +++ b/grafana/dashboards/mysql/DORA.json @@ -43,7 +43,7 @@ "content": "- See [how to config](https://devlake.apache.org/docs/DORA) this dashboard\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- You can validate/debug this dashboard with the [DORA validation dashboard](/grafana/d/KGkUnV-Vz/dora-dashboard-validation)\n- DORA benchmarks vary in different years. You can switch the benchmarks to change them.\n- In DORA's official report in 2023, metric 'failed deployment recovery time' has replaced 'MTTR'.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -58,7 +58,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -223,10 +223,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -278,7 +278,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "How often an organization deploys code to production or release it to end users\n\n# 2023 Benchmarks\n- **Elite**: >= 7 deployment days per week\n- **High**: >= 1 deployment day per week\n- **Medium**: >= 1 deployment day per month\n- **Low**: < 1 deployment day per month\n# 2021 Benchmarks\n- **Elite**: >= 7 deployment days per week\n- **High**: >= 1 deployment day per month\n- **Medium**: >= 1 deployment day per six months\n- **Low**: < 1 deployment day per six months", "fieldConfig": { "defaults": { @@ -370,10 +370,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -425,7 +425,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "The median amount of time for a code change to be deployed into production.\n\n\n# 2023 Benchmarks\n- **Elite**: Less than one day\n- **High**: Between one day and one week\n- **Medium**: Between one week and one month\n- **Low**: More than one month\n# 2021 Benchmarks\n- **Elite**: Less than one hour\n- **High**: Less than one week\n- **Medium**: Between one week and six months\n- **Low**: More than six months\n", "fieldConfig": { "defaults": { @@ -516,10 +516,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -571,7 +571,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "The percentage of changes that were made to a code that then resulted in incidents, rollbacks, or any type of production failure.\n\n# 2023 Benchmarks\n- **Elite**: (0, 5%]\n- **High**: (5%, 10%]\n- **Medium**: (10%, 15%]\n- **Low**: (15%, 100%]\n# 2021 Benchmarks\n- **Elite**: (0, 15%]\n- **High**: (16%, 20%]\n- **Medium**: (21%, 30%]\n- **Low**: (30%, 100%]\n", "fieldConfig": { "defaults": { @@ -663,10 +663,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -718,7 +718,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "# MTTR benchmarks:\ndevlake.apache.org/docs/Metrics/MTTR#how-is-it-calculated\n# FDRT benchmarks:\ndevlake.apache.org/docs/Metrics/FailedDeploymentRecoveryTime#how-is-it-calculated\n", "fieldConfig": { "defaults": { @@ -817,10 +817,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -872,7 +872,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -955,7 +955,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1007,7 +1007,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1085,10 +1085,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1140,7 +1140,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1239,10 +1239,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1294,7 +1294,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1397,10 +1397,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1471,7 +1471,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -1492,7 +1492,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -1513,7 +1513,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/mysql/GithubCopilotREADME.md b/grafana/dashboards/mysql/GithubCopilotREADME.md new file mode 100644 index 00000000000..642ff425296 --- /dev/null +++ b/grafana/dashboards/mysql/GithubCopilotREADME.md @@ -0,0 +1,127 @@ + +# GitHub Copilot Dashboards + +Grafana dashboards for analyzing GitHub Copilot usage and its correlation with developer productivity metrics. + +## Dashboards + +### 1. GitHub Copilot Adoption Dashboard (`GithubCopilotAdoption.json`) + +**UID**: `copilot_adoption` + +Tracks GitHub Copilot usage metrics across your organization: +- Active users and seat utilization +- Language breakdown of GitHub Copilot activity +- IDE distribution (VS Code, JetBrains, Neovim, etc.) +- Acceptance rates for code suggestions +- Chat and PR summary feature usage + +### 2. GitHub Copilot Impact Dashboard (`GithubCopilotImpact.json`) + +**UID**: `copilot_impact` + +Correlates GitHub Copilot adoption with DORA metrics and engineering productivity: + +#### Key Features + +- **Correlation-First Analysis**: No implementation date required! Dashboard automatically correlates GitHub Copilot adoption intensity with productivity metrics. +- **Adoption Tier Comparison**: Groups metrics by adoption level (<25%, 25-50%, 50-75%, >75%) +- **Dual-Axis Charts**: See GitHub Copilot adoption trends alongside each DORA metric +- **Pearson Correlation Coefficients**: Statistical correlation (r) values for each metric pair +- **Optional Rollout Milestone**: Annotate specific dates when GitHub Copilot was rolled out to different teams + +#### Panels by Section + +| Section | Metrics Tracked | +|---------|-----------------| +| Correlation Overview | Adoption trend, aggregate correlation, current adoption % | +| PR Velocity Impact | PR cycle time, coding time, pickup time, review time, PR throughput | +| Deployment Frequency | Deploys per week, correlation with adoption | +| Change Failure Rate | CFR %, correlation (negative r = improvement) | +| Recovery Time (MTTR) | Mean time to recovery, adoption tier comparison | +| Code Review Time | Review time by adoption tier, trend analysis | +| Code Quality | Optional (requires SonarQube): complexity, coverage, duplicates | + +#### Correlation Interpretation + +The dashboard uses Pearson correlation coefficients (r): + +| Value | Interpretation | +|-------|----------------| +| r > 0.7 | Strong positive correlation | +| 0.3 < r < 0.7 | Moderate correlation | +| r < 0.3 | Weak correlation | +| r < 0 | Negative correlation (for CFR/MTTR, negative = improvement) | + +**Note**: For failure-related metrics (CFR, MTTR, Review Time), **negative** correlations are desirable - they indicate that higher GitHub Copilot adoption correlates with fewer failures or faster resolution. + +## Prerequisites + +These dashboards require: + +1. **GitHub Copilot Plugin** (`gh-copilot`) - Configured and collecting data +2. **GitHub Plugin** - For PR metrics (`project_pr_metrics` table) +3. **DORA Metrics** - Deployments and incidents data from your CI/CD tools + +## Configuration + +### Variables + +Both dashboards use these template variables: + +| Variable | Description | +|----------|-------------| +| `connection_id` | DevLake GitHub Copilot connection | +| `scope_id` | Enterprise/Organization scope | +| `project` | DevLake project filter | + +### Optional: Rollout Milestone + +To add an implementation date annotation: + +1. Go to **Connections** > **GitHub Copilot** > Edit Scope +2. Set **Implementation Date** (when GitHub Copilot was rolled out) +3. Set **Baseline Period** (days to use for "before" comparison) + +The dashboard works fully without these settings—correlation analysis doesn't require an implementation date. + +## Data Model + +The Impact Dashboard joins GitHub Copilot metrics with DORA data: + +```sql +-- Weekly aggregation pattern +_copilot_adoption → _adoption_weekly (by week_start) + → JOIN with metric CTEs (pr_metrics, deployments, incidents) +``` + +Key tables used: +- `_tool_copilot_org_metrics` - Daily GitHub Copilot usage +- `project_pr_metrics` - PR cycle time, review time +- `cicd_deployment_commits` - Deployment frequency, CFR +- `issues` (type='INCIDENT') - MTTR calculation +- `cq_file_metrics` - Code quality (SonarQube) + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| "No data" on panels | Check that time range includes GitHub Copilot data; verify `scope_id` variable | +| Correlation shows "N/A" | Need at least 2 weeks of overlapping data | +| CFR/MTTR empty | Ensure incidents are collected (Jira/GitHub issues with 'incident' label) | +| Code Quality empty | Configure SonarQube integration | diff --git a/grafana/dashboards/Gitlab.json b/grafana/dashboards/mysql/Gitlab.json similarity index 95% rename from grafana/dashboards/Gitlab.json rename to grafana/dashboards/mysql/Gitlab.json index a45bb37f6a7..3a7043b69ec 100644 --- a/grafana/dashboards/Gitlab.json +++ b/grafana/dashboards/mysql/Gitlab.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from GitLab.\n- Data Source Required: GitLab", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -91,7 +91,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -140,10 +140,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -195,7 +195,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -290,10 +290,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -345,7 +345,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -429,10 +429,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -506,7 +506,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -558,7 +558,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -594,7 +594,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -718,10 +718,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -773,7 +773,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -822,10 +822,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -877,7 +877,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -959,10 +959,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1014,7 +1014,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1068,7 +1068,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1104,7 +1104,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1188,10 +1188,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1256,7 +1256,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of pipeline runs executed in the selected time range", "fieldConfig": { "defaults": { @@ -1303,10 +1303,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1357,7 +1357,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful pipeline runs / Number of total pipeline runs", "fieldConfig": { "defaults": { @@ -1410,10 +1410,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1464,7 +1464,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1577,10 +1577,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1632,7 +1632,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1765,10 +1765,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1819,7 +1819,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -1906,10 +1906,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1960,7 +1960,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful pipeline runs / Number of total pipeline runs", "fieldConfig": { "defaults": { @@ -2007,10 +2007,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2061,7 +2061,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2161,10 +2161,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2236,7 +2236,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2270,7 +2270,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos where id like 'gitlab%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Homepage.json b/grafana/dashboards/mysql/Homepage.json similarity index 96% rename from grafana/dashboards/Homepage.json rename to grafana/dashboards/mysql/Homepage.json index 2332eeb01e6..f4f499e6902 100644 --- a/grafana/dashboards/Homepage.json +++ b/grafana/dashboards/mysql/Homepage.json @@ -43,7 +43,7 @@ "content": "# Welcome\n\n - Dashboards are grouped by use cases and data sources.\n - All data is queried from here. You can go to the 'Explore' page on the left menu to play around.\n - You can always make your own dashboard based on the domain layer schema, SQL examples in engineering metrics, and dashboard manuals.\n", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -78,7 +78,7 @@ "showStarred": true, "tags": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "Highlights" ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -179,7 +179,7 @@ "Engineering Leads Dashboard" ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "title": "For Engineering Leads", "type": "dashlist" }, @@ -206,7 +206,7 @@ "OSS Maintainer Dashboard" ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "title": "For OSS Maintainers", "type": "dashlist" }, @@ -233,7 +233,7 @@ "Developer" ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "title": "For Developers", "type": "dashlist" }, @@ -286,7 +286,7 @@ "Data Source Dashboard" ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "type": "dashlist" } ], diff --git a/grafana/dashboards/Jenkins.json b/grafana/dashboards/mysql/Jenkins.json similarity index 95% rename from grafana/dashboards/Jenkins.json rename to grafana/dashboards/mysql/Jenkins.json index 7c1a9db8138..6b39ae9988c 100644 --- a/grafana/dashboards/Jenkins.json +++ b/grafana/dashboards/mysql/Jenkins.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from Jenkins, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate).\n- Data Source Required: Jenkins", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -65,7 +65,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of builds executed in the selected time range", "fieldConfig": { "defaults": { @@ -111,10 +111,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -165,7 +165,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful builds / Number of total builds", "fieldConfig": { "defaults": { @@ -214,10 +214,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -268,7 +268,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -401,10 +401,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -455,7 +455,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful builds / Number of total builds", "fieldConfig": { "defaults": { @@ -502,10 +502,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -538,7 +538,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -621,10 +621,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -676,7 +676,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -779,10 +779,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -833,7 +833,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -946,10 +946,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -983,7 +983,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1082,10 +1082,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1139,7 +1139,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1172,7 +1172,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from cicd_scopes where id like \"jenkins%\" ", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Jira.json b/grafana/dashboards/mysql/Jira.json similarity index 94% rename from grafana/dashboards/Jira.json rename to grafana/dashboards/mysql/Jira.json index a18e46b3f27..ad663e498aa 100644 --- a/grafana/dashboards/Jira.json +++ b/grafana/dashboards/mysql/Jira.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the basic project management metrics from Jira.\n- Data Source Required: Jira", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -167,10 +167,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -220,7 +220,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -272,10 +272,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -325,7 +325,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -411,10 +411,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -446,7 +446,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -502,10 +502,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -537,7 +537,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -621,10 +621,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -682,7 +682,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -735,10 +735,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -789,7 +789,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -841,10 +841,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -877,7 +877,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -964,10 +964,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1000,7 +1000,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", "fieldConfig": { "defaults": { @@ -1081,10 +1081,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 100 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1165,7 +1165,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1196,7 +1196,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'jira%'", "hide": 0, "includeAll": true, @@ -1217,7 +1217,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Opsgenie.json b/grafana/dashboards/mysql/Opsgenie.json similarity index 95% rename from grafana/dashboards/Opsgenie.json rename to grafana/dashboards/mysql/Opsgenie.json index 2ab5a9e62b0..627d412bb8b 100644 --- a/grafana/dashboards/Opsgenie.json +++ b/grafana/dashboards/mysql/Opsgenie.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the incident data from Opsgenie.\n- Data Source Required: Opsgenie", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -118,7 +118,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Opsgenie board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -167,10 +167,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -220,7 +220,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -270,10 +270,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -323,7 +323,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -402,10 +402,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -455,7 +455,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -505,10 +505,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -558,7 +558,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -612,10 +612,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -665,7 +665,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -746,10 +746,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -825,7 +825,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -881,10 +881,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -935,7 +935,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -990,10 +990,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1044,7 +1044,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1129,10 +1129,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1187,7 +1187,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of MTTR\n2. Each point refers to the percent rank of a distinct duration to resolve incidents.", "fill": 0, "fillGradient": 4, @@ -1224,7 +1224,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "9.5.2", + "pluginVersion": "13.0.2", "pointradius": 0.5, "points": false, "renderer": "flot", @@ -1234,7 +1234,7 @@ "steppedLine": false, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1349,7 +1349,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.2", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1376,7 +1376,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Due to Opsgenie API limitation regarding the relationship between assignee, users, and incident (and lack of 'assign at' key ), the column assignee represents a JSON object containing all assignee from said incident", "fieldConfig": { "defaults": { @@ -1458,10 +1458,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1506,7 +1506,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'opsgenie%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Sonarqube.json b/grafana/dashboards/mysql/Sonarqube.json similarity index 72% rename from grafana/dashboards/Sonarqube.json rename to grafana/dashboards/mysql/Sonarqube.json index d8708f36d93..4ab5445766d 100644 --- a/grafana/dashboards/Sonarqube.json +++ b/grafana/dashboards/mysql/Sonarqube.json @@ -46,10 +46,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard shows the code quality metrics from SonarQube.\n- Data Source Required: SonarQube v8.2+\n- This dashboard does not honor the time filter on the top-right side as SonarQube metrics are all from the latest scan.", + "content": "- Use Cases: This dashboard shows the code quality metrics from SonarQube.\n- Data Source Required: SonarQube v8.2+\n- Snapshot panels (Reliability, Security, Test, etc.) show the latest scan and do not honor the time filter.\n- **Historical Trends** panels at the bottom respond to the time range selector and show metric history over time.", "mode": "markdown" }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -88,7 +88,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -144,10 +144,10 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -176,7 +176,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -232,10 +232,10 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -264,7 +264,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -321,10 +321,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -353,7 +353,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -409,10 +409,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -467,7 +467,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -523,10 +523,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -555,7 +555,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -612,10 +612,10 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -644,7 +644,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -701,10 +701,10 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -759,7 +759,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -815,10 +815,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -847,7 +847,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -903,10 +903,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -935,7 +935,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -993,10 +993,10 @@ }, "showHeader": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1023,6 +1023,263 @@ ], "title": "Code Quality Metrics by Files (Top 20 order by Bugs)", "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 20, + "panels": [ + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 21, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.coverage AS 'Coverage %'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Coverage Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 22, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.bugs AS 'Bugs',\n pmh.vulnerabilities AS 'Vulnerabilities'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Bugs & Vulnerabilities Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 23, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.code_smells AS 'Code Smells'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Code Smells Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 24, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.duplicated_lines_density AS 'Duplication %'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Duplication Trend", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Historical Trends", + "type": "row" } ], "refresh": "", @@ -1043,7 +1300,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from cq_projects", "hide": 0, "includeAll": true, @@ -1068,7 +1325,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct severity from cq_issues where id like 'sonar%'", "hide": 0, "includeAll": true, @@ -1086,7 +1343,7 @@ ] }, "time": { - "from": "now", + "from": "now-6M", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/TAPD.json b/grafana/dashboards/mysql/TAPD.json similarity index 94% rename from grafana/dashboards/TAPD.json rename to grafana/dashboards/mysql/TAPD.json index da960da5293..2fabb39a510 100644 --- a/grafana/dashboards/TAPD.json +++ b/grafana/dashboards/mysql/TAPD.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the basic project management metrics from TAPD.\n- Data Source Required: TAPD", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -165,10 +165,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -218,7 +218,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -268,10 +268,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -321,7 +321,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -404,10 +404,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -439,7 +439,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -493,10 +493,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -528,7 +528,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -609,10 +609,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -670,7 +670,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -721,10 +721,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -757,7 +757,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -807,10 +807,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -843,7 +843,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -928,10 +928,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -968,7 +968,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", "fill": 0, "fillGradient": 4, @@ -1005,7 +1005,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "pointradius": 0.5, "points": false, "renderer": "flot", @@ -1015,7 +1015,7 @@ "steppedLine": false, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -1112,7 +1112,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1144,7 +1144,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'tapd%'", "hide": 0, "includeAll": true, @@ -1165,7 +1165,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/mysql/Taiga.json b/grafana/dashboards/mysql/Taiga.json new file mode 100644 index 00000000000..454f62722da --- /dev/null +++ b/grafana/dashboards/mysql/Taiga.json @@ -0,0 +1,2316 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [ + { + "targetBlank": true, + "title": "Taiga Plugin", + "url": "https://github.com/apache/incubator-devlake" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows project management metrics from Taiga across all issue types (User Stories, Tasks, Issues/Requirements, Epics).\n- Data Source Required: Taiga\n- Metrics: Issue throughput (filtered by type), lead time, story points (User Stories), milestone progress, assignee breakdown and issue type breakdown.\n- Use the **Issue Type** variable to filter panels to a specific type, or select All to see all types together.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 100, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total number of user stories created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select\r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total number of issues that have been marked as Done/Closed in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 3, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "select\r\n count(distinct i.id) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Closed Issues [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Monthly distribution of open vs. closed issues.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n count(distinct case when status != 'DONE' then i.id else null end) as \"Number of Open Stories\",\r\n count(distinct case when status = 'DONE' then i.id else null end) as \"Number of Closed Stories\"\r\nFROM issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Issue Delivery Rate = count(Closed Issues) / count(All Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 5, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n count(distinct i.id) as total_count,\r\n count(distinct case when i.status = 'DONE' then i.id else null end) as delivered_count\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n now() as time,\r\n 1.0 * delivered_count/total_count as requirement_delivery_rate\r\nfrom _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Monthly trend of issue delivery rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\r\n 1.0 * count(distinct case when i.status = 'DONE' then i.id else null end)/count(distinct i.id) as delivered_rate\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n time,\r\n delivered_rate\r\nfrom _requirements\r\norder by time", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 101, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average number of days from story creation to resolution.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 7, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n avg(lead_time_minutes/1440) as value\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Lead Time in Days [Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "80% of stories are resolved within this many days.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select\r\n i.lead_time_minutes,\r\n percent_rank() over (order by lead_time_minutes asc) as ranks\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n)\r\n\r\nselect\r\n max(lead_time_minutes/1440) as value\r\nfrom _ranks\r\nwhere\r\n ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Stories' Lead Time ≤ # Days [Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Mean lead time per month for resolved stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time (days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _requirements as(\r\n select\r\n DATE_ADD(date(i.resolution_date), INTERVAL -DAYOFMONTH(date(i.resolution_date))+1 DAY) as time,\r\n avg(lead_time_minutes/1440) as mean_lead_time\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n group by 1\r\n)\r\n\r\nselect\r\n date_format(time,'%M %Y') as month,\r\n mean_lead_time\r\nfrom _requirements\r\norder by time asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Lead Time by Month [Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "The cumulative distribution of story lead time. Each point shows the percent rank of a given lead time value.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "with _ranks as(\r\n select\r\n round(i.lead_time_minutes/1440) as lead_time_day\r\n from issues i\r\n join board_issues bi on i.id = bi.issue_id\r\n where\r\n i.type in (${type})\r\n and i.status = 'DONE'\r\n and $__timeFilter(i.resolution_date)\r\n and bi.board_id in (${board_id})\r\n order by lead_time_day asc\r\n)\r\n\r\nselect\r\n now() as time,\r\n lpad(concat(lead_time_day,'d'), 4, ' ') as metric,\r\n percent_rank() over (order by lead_time_day asc) as value\r\nfrom _ranks\r\norder by lead_time_day asc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Lead Time [Resolved in Selected Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 102, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "3. Story Points", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total story points for all user stories in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 30 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n sum(ts.total_points) as value\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\nwhere\r\n bi.board_id in (${board_id})", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Story Points", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total story points for closed/completed user stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 30 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n sum(ts.total_points) as value\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\n join issues i on i.id = bi.issue_id\r\nwhere\r\n bi.board_id in (${board_id})\r\n and i.status = 'DONE'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Completed Story Points", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Story points allocated to each milestone/sprint. Requires milestone data from Taiga.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Story Points", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 30 + }, + "id": 13, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n COALESCE(NULLIF(ts.milestone_name,''), 'No Milestone') as Milestone,\r\n sum(ts.total_points) as `Story Points`\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\nwhere\r\n bi.board_id in (${board_id})\r\ngroup by 1\r\norder by `Story Points` desc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points by Milestone", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Story points distributed across assignees.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Story Points", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 14, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n COALESCE(NULLIF(i.assignee_name,''), 'Unassigned') as Assignee,\r\n sum(i.story_point) as `Story Points`\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1\r\norder by `Story Points` desc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points by Assignee [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of story points across open vs. closed stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 15, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n sum(ts.total_points) as `Story Points`,\r\n CASE WHEN i.status = 'DONE' THEN 'Completed' ELSE 'In Progress' END as Status\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\n join issues i on i.id = bi.issue_id\r\nwhere\r\n bi.board_id in (${board_id})\r\ngroup by Status", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points: Completed vs In Progress", + "type": "piechart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 103, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "4. Issues Overview", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Breakdown of all Taiga issues by their current status, across all issue types (User Story, Task, Issue/Requirement, Epic).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 16, + "options": { + "displayLabels": ["name", "value"], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n now() as time,\n count(distinct case when i.status = 'TODO' then i.id end) as 'To Do',\n count(distinct case when i.status = 'IN_PROGRESS' then i.id end) as 'In Progress',\n count(distinct case when i.status = 'DONE' then i.id end) as 'Done',\n count(distinct case when i.status not in ('TODO','IN_PROGRESS','DONE') then i.id end) as 'Other'\nFROM issues i\n join board_issues bi on i.id = bi.issue_id\nWHERE bi.board_id in (${board_id})", + "refId": "A" + } + ], + "title": "Issues by Status", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Number of user stories assigned to each team member.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "User Stories", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "id": 17, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n COALESCE(NULLIF(i.assignee_name,''), 'Unassigned') as Assignee,\r\n count(distinct i.id) as `User Stories`\r\nfrom issues i\r\n join board_issues bi on i.id = bi.issue_id\r\nwhere\r\n i.type in (${type})\r\n and $__timeFilter(i.created_date)\r\n and bi.board_id in (${board_id})\r\ngroup by 1\r\norder by `User Stories` desc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issues by Assignee [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Number of user stories in each milestone/sprint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "User Stories", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 18, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n COALESCE(NULLIF(ts.milestone_name,''), 'No Milestone') as Milestone,\r\n count(distinct ts.user_story_id) as `User Stories`\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\nwhere\r\n bi.board_id in (${board_id})\r\ngroup by 1\r\norder by `User Stories` desc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "User Stories per Milestone", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Milestone progress: open vs. closed user stories per sprint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 19, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "select\r\n COALESCE(NULLIF(ts.milestone_name,''), 'No Milestone') as Milestone,\r\n count(distinct case when i.status != 'DONE' then ts.user_story_id else null end) as `Open Stories`,\r\n count(distinct case when i.status = 'DONE' then ts.user_story_id else null end) as `Closed Stories`\r\nfrom _tool_taiga_user_stories ts\r\n join board_issues bi on bi.issue_id = concat('taiga:TaigaUserStory:', ts.connection_id, ':', ts.user_story_id)\r\n join issues i on i.id = bi.issue_id\r\nwhere\r\n bi.board_id in (${board_id})\r\ngroup by 1\r\norder by `Closed Stories` desc", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Milestone Progress: Open vs. Closed Stories", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 104, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "5. Issue Type Breakdown", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of all Taiga items across issue types (User Story, Task, Issue/Requirement, Epic).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "hideFrom": { "legend": false, "tooltip": false, "viz": false } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 6, "x": 0, "y": 61 }, + "id": 20, + "options": { + "displayLabels": ["name", "value"], + "legend": { "displayMode": "list", "placement": "right", "showLegend": true }, + "pieType": "pie", + "tooltip": { "mode": "single", "sort": "none" } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n now() as time,\n count(distinct case when i.type = 'USER_STORY' then i.id end) as 'User Story',\n count(distinct case when i.type = 'TASK' then i.id end) as 'Task',\n count(distinct case when i.type = 'REQUIREMENT' then i.id end) as 'Issue',\n count(distinct case when i.type = 'EPIC' then i.id end) as 'Epic'\nFROM issues i\n join board_issues bi on i.id = bi.issue_id\nWHERE bi.board_id in (${board_id})", + "refId": "A" + } + ], + "title": "Issues by Type", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Open vs closed count for each Taiga issue type in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 10, "x": 6, "y": 61 }, + "id": 21, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { "mode": "multi", "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select\n COALESCE(i.original_type, i.type) as `Issue Type`,\n count(distinct case when i.status != 'DONE' then i.id else null end) as `Open`,\n count(distinct case when i.status = 'DONE' then i.id else null end) as `Closed`\nfrom issues i\n join board_issues bi on i.id = bi.issue_id\nwhere\n $__timeFilter(i.created_date)\n and bi.board_id in (${board_id})\ngroup by 1\norder by 1", + "refId": "A" + } + ], + "title": "Open vs Closed by Type [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Monthly count of items created, broken down by issue type (User Stories, Tasks, Issues, Epics).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 80 }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 61 }, + "id": 22, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_ADD(date(i.created_date), INTERVAL -DAYOFMONTH(date(i.created_date))+1 DAY) as time,\n count(distinct case when i.type='USER_STORY' then i.id end) as 'User Stories',\n count(distinct case when i.type='TASK' then i.id end) as 'Tasks',\n count(distinct case when i.type='REQUIREMENT' then i.id end) as 'Issues',\n count(distinct case when i.type='EPIC' then i.id end) as 'Epics'\nFROM issues i\n join board_issues bi on i.id = bi.issue_id\nWHERE\n bi.board_id in (${board_id})\n AND $__timeFilter(i.created_date)\nGROUP BY 1\nORDER BY 1", + "refId": "A" + } + ], + "title": "Items Created per Month by Type", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average lead time in days from creation to resolution, broken down by issue type (resolved issues only).", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Avg Lead Time (days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }, { "color": "red", "value": 80 }] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 69 }, + "id": 23, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select\n COALESCE(i.original_type, i.type) as `Issue Type`,\n round(avg(\n COALESCE(i.lead_time_minutes, TIMESTAMPDIFF(MINUTE, i.created_date, i.resolution_date))\n /1440), 1) as `Avg Lead Time (days)`\nfrom issues i\n join board_issues bi on i.id = bi.issue_id\nwhere\n i.status = 'DONE'\n and bi.board_id in (${board_id})\ngroup by 1\nhaving `Avg Lead Time (days)` is not null\norder by 2 desc", + "refId": "A" + } + ], + "title": "Average Lead Time by Type", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select concat(name, '--', id) from boards where id like 'taiga%'", + "hide": 0, + "includeAll": true, + "label": "Choose Project", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'taiga%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "select distinct i.type from issues i join board_issues bi on i.id = bi.issue_id where bi.board_id like 'taiga%'", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": true, + "name": "type", + "options": [], + "query": "select distinct i.type from issues i join board_issues bi on i.id = bi.issue_id where bi.board_id like 'taiga%'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Taiga", + "uid": "taiga-dashboard", + "version": 2, + "weekStart": "" +} diff --git a/grafana/dashboards/Teambition.json b/grafana/dashboards/mysql/Teambition.json similarity index 94% rename from grafana/dashboards/Teambition.json rename to grafana/dashboards/mysql/Teambition.json index 743884e1540..3bbda241f8e 100644 --- a/grafana/dashboards/Teambition.json +++ b/grafana/dashboards/mysql/Teambition.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the basic project management metrics from Teambition.\n- Data Source Required: Teambition", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -165,10 +165,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -218,7 +218,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -268,10 +268,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -321,7 +321,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -404,10 +404,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -439,7 +439,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -493,10 +493,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -528,7 +528,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -609,10 +609,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -670,7 +670,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -721,10 +721,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -757,7 +757,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -807,10 +807,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -843,7 +843,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -928,10 +928,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -968,7 +968,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", "fill": 0, "fillGradient": 4, @@ -1005,7 +1005,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "pointradius": 0.5, "points": false, "renderer": "flot", @@ -1015,7 +1015,7 @@ "steppedLine": false, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -1112,7 +1112,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1144,7 +1144,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'teambition%'", "hide": 0, "includeAll": true, @@ -1165,7 +1165,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Testmo.json b/grafana/dashboards/mysql/Testmo.json similarity index 94% rename from grafana/dashboards/Testmo.json rename to grafana/dashboards/mysql/Testmo.json index 934f13e7562..2cf6c751962 100644 --- a/grafana/dashboards/Testmo.json +++ b/grafana/dashboards/mysql/Testmo.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows comprehensive test management metrics from Testmo.\n- Data Source Required: Testmo\n- Metrics: Test execution, success rates, coverage, and performance analytics", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Total number of test runs executed in the selected time range", "fieldConfig": { "defaults": { @@ -158,10 +158,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -176,7 +176,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Percentage of test runs that passed successfully", "fieldConfig": { "defaults": { @@ -227,10 +227,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -245,7 +245,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Test execution results over time showing pass/fail/skip distribution", "fieldConfig": { "defaults": { @@ -363,10 +363,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -380,7 +380,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Total number of Testmo runs (new data source)", "fieldConfig": { "defaults": { @@ -422,10 +422,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -439,7 +439,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Average execution time per test run in minutes", "fieldConfig": { "defaults": { @@ -491,10 +491,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -508,7 +508,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Daily count of test runs from the new runs data source", "fieldConfig": { "defaults": { @@ -599,7 +599,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -613,7 +613,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Test coverage across different projects", "fieldConfig": { "defaults": { @@ -684,7 +684,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -723,7 +723,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Success rate trend over time", "fieldConfig": { "defaults": { @@ -797,7 +797,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -810,7 +810,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Average test execution time trend", "fieldConfig": { "defaults": { @@ -882,7 +882,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -895,7 +895,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Comparison between automation runs and regular test runs over time", "fieldConfig": { "defaults": { @@ -1001,7 +1001,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -1010,7 +1010,7 @@ "refId": "A" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -1023,7 +1023,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Test runs that take the longest time to execute", "fieldConfig": { "defaults": { @@ -1113,10 +1113,10 @@ }, "showHeader": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1149,7 +1149,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1181,7 +1181,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from _tool_testmo_connections", "hide": 0, "includeAll": true, @@ -1202,7 +1202,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from _tool_testmo_projects where connection_id in (${connection_id})", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/Zentao.json b/grafana/dashboards/mysql/Zentao.json similarity index 94% rename from grafana/dashboards/Zentao.json rename to grafana/dashboards/mysql/Zentao.json index dbde7eb46bf..009259e8b84 100644 --- a/grafana/dashboards/Zentao.json +++ b/grafana/dashboards/mysql/Zentao.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the basic project management metrics from Zentao.\n- Data Source Required: Zentao", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -165,10 +165,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -218,7 +218,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -268,10 +268,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -321,7 +321,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -404,10 +404,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -439,7 +439,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -493,10 +493,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -528,7 +528,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", "fieldConfig": { "defaults": { @@ -609,10 +609,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -670,7 +670,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -721,10 +721,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -757,7 +757,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -807,10 +807,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -843,7 +843,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -928,10 +928,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -968,7 +968,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", "fill": 0, "fillGradient": 4, @@ -1005,7 +1005,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "pointradius": 0.5, "points": false, "renderer": "flot", @@ -1015,7 +1015,7 @@ "steppedLine": false, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -1112,7 +1112,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1144,7 +1144,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'zentao%'", "hide": 0, "includeAll": true, @@ -1165,7 +1165,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/mysql/ai-cost-efficiency.json b/grafana/dashboards/mysql/ai-cost-efficiency.json new file mode 100644 index 00000000000..d52905d0cca --- /dev/null +++ b/grafana/dashboards/mysql/ai-cost-efficiency.json @@ -0,0 +1,317 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 1 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(credits_used)) AS 'Total Credits'\nFROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits Used", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 6, "y": 1 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT pr.id), 0), 1) AS 'Credits / PR'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n) pr\nWHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per PR (Overall)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 12, "y": 1 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 1) AS 'Credits / Deploy'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT cicd_deployment_id\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n) cdc\nWHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Deployment (Overall)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 18, "y": 1 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(COUNT(DISTINCT i.id), 0), 1) AS 'Credits / Issue'\nFROM _tool_q_dev_user_report r\nCROSS JOIN (\n SELECT DISTINCT id FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n) i\nWHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly cost per merged PR", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 11, + "options": { + "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_prs AS (\n SELECT DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY) AS week_start,\n COUNT(*) AS prs\n FROM pull_requests\n WHERE merged_date IS NOT NULL AND $__timeFilter(merged_date)\n GROUP BY DATE_SUB(DATE(merged_date), INTERVAL WEEKDAY(DATE(merged_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(p.prs, 0), 1) AS 'Credits per PR'\nFROM _credits c\nLEFT JOIN _prs p ON c.week_start = p.week_start\nORDER BY time", + "refId": "A" + } + ], + "title": "Credits per Merged PR (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly cost per production deployment", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 12, + "options": { + "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploys AS (\n SELECT DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(d.deploys, 0), 1) AS 'Credits per Deploy'\nFROM _credits c\nLEFT JOIN _deploys d ON c.week_start = d.week_start\nORDER BY time", + "refId": "A" + } + ], + "title": "Credits per Deployment (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly cost per resolved issue", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, + "id": 13, + "options": { + "legend": { "calcs": ["mean", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_issues AS (\n SELECT DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n COUNT(*) AS resolved\n FROM issues\n WHERE resolution_date IS NOT NULL AND type != 'INCIDENT'\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT c.week_start AS time,\n ROUND(c.credits / NULLIF(i.resolved, 0), 1) AS 'Credits per Issue'\nFROM _credits c\nLEFT JOIN _issues i ON c.week_start = i.week_start\nORDER BY time", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Is cost efficiency improving over time?", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, + "id": 14, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits',\n SUM(total_messages) AS 'Messages',\n SUM(chat_conversations) AS 'Conversations'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly AI Activity Volume", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "kiro", "cost", "efficiency"], + "templating": { "list": [] }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "AI Cost-Efficiency", + "uid": "ai_cost_efficiency", + "version": 1 +} diff --git a/grafana/dashboards/mysql/ai-model-roi.json b/grafana/dashboards/mysql/ai-model-roi.json new file mode 100644 index 00000000000..49980a1fadb --- /dev/null +++ b/grafana/dashboards/mysql/ai-model-roi.json @@ -0,0 +1,537 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits Used", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per PR (Overall)", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Deployment (Overall)", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per merged PR", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Merged PR (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per production deployment", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Deployment (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per resolved issue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Is cost efficiency improving over time?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly AI Activity Volume", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "cost", + "efficiency" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "AI Cost-Efficiency (PostgreSQL)", + "uid": "ai_cost_efficiency-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/ArgoCD.json b/grafana/dashboards/mysql/argo-cd.json similarity index 91% rename from grafana/dashboards/ArgoCD.json rename to grafana/dashboards/mysql/argo-cd.json index e3fa3956ec9..1f589224ce0 100644 --- a/grafana/dashboards/ArgoCD.json +++ b/grafana/dashboards/mysql/argo-cd.json @@ -50,7 +50,7 @@ "content": "- Use Cases: Track ArgoCD deployment activity, measure success rates, monitor environments, and spot slow deployments.\n- Data Source Required: ArgoCD plugin (deployments mapped into `cicd_deployments`).\n- Filters: Use the *Application* templating variable to focus on specific ArgoCD applications.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -65,7 +65,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -103,10 +103,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -120,7 +120,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -158,10 +158,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -175,7 +175,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Successful deployments / Total deployments", "fieldConfig": { "defaults": { @@ -217,10 +217,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -234,7 +234,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -284,10 +284,10 @@ "sort": "none" } }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -314,7 +314,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -361,10 +361,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -378,7 +378,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -429,10 +429,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -446,7 +446,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -493,10 +493,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -523,7 +523,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Average duration of deployments (minutes)", "fieldConfig": { "defaults": { @@ -563,10 +563,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -580,7 +580,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -627,10 +627,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -644,7 +644,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -695,10 +695,10 @@ } ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -712,7 +712,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "7-day rolling average of deployment duration (minutes)", "fieldConfig": { "defaults": { @@ -749,10 +749,10 @@ "sort": "none" } }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -779,7 +779,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -830,10 +830,10 @@ } ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -860,7 +860,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "mappings": [], @@ -901,10 +901,10 @@ } ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -918,7 +918,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Approximation: counts deployments per unique images array (revision-level). For per-image counts, consider exploding arrays during ingestion.", "fieldConfig": { "defaults": { @@ -966,10 +966,10 @@ "sort": "none" } }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1002,7 +1002,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "allValue": "$__all", "definition": "select concat(name, '--', id) as text from cicd_scopes where id like 'argocd:ArgocdApplication:%'", "hide": 0, diff --git a/grafana/dashboards/AzureDevOps.json b/grafana/dashboards/mysql/azure-dev-ops.json similarity index 95% rename from grafana/dashboards/AzureDevOps.json rename to grafana/dashboards/mysql/azure-dev-ops.json index 3f8f00f98b1..486d7b07f14 100644 --- a/grafana/dashboards/AzureDevOps.json +++ b/grafana/dashboards/mysql/azure-dev-ops.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from Azure DevOps.\n- Data Source Required: Azure DevOps", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -91,7 +91,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -141,10 +141,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -196,7 +196,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -291,10 +291,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -346,7 +346,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -430,10 +430,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -511,7 +511,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -563,10 +563,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -600,7 +600,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -724,10 +724,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -779,7 +779,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -828,10 +828,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -883,7 +883,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -965,10 +965,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1020,7 +1020,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1074,10 +1074,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1111,7 +1111,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1195,10 +1195,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1263,7 +1263,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of Pipeline runs executed in the selected time range", "fieldConfig": { "defaults": { @@ -1310,10 +1310,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1364,7 +1364,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful Pipeline runs / Number of total Pipeline runs", "fieldConfig": { "defaults": { @@ -1417,10 +1417,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1471,7 +1471,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1584,10 +1584,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1639,7 +1639,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful Pipeline runs / Number of total Pipeline runs", "fieldConfig": { "defaults": { @@ -1772,10 +1772,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1826,7 +1826,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -1913,10 +1913,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1967,7 +1967,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Pipeline runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"Pipeline runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -2014,10 +2014,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2068,7 +2068,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2168,10 +2168,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2243,7 +2243,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2277,7 +2277,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos where id like 'azure%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/BitBucket.json b/grafana/dashboards/mysql/bit-bucket.json similarity index 95% rename from grafana/dashboards/BitBucket.json rename to grafana/dashboards/mysql/bit-bucket.json index 4f281c67f99..e5ef78adf74 100644 --- a/grafana/dashboards/BitBucket.json +++ b/grafana/dashboards/mysql/bit-bucket.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from Bitbucket.\n- Data Source Required: Bitbucket Cloud or Server/data center. Please note that for Bitbucket Server/Data center, there is no embedded CI/CD data.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -91,7 +91,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -144,10 +144,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -199,7 +199,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -294,10 +294,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -349,7 +349,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -433,10 +433,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -514,7 +514,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -566,10 +566,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -603,7 +603,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -724,10 +724,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -779,7 +779,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -828,10 +828,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -883,7 +883,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -965,10 +965,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1020,7 +1020,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1074,10 +1074,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1111,7 +1111,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1195,10 +1195,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1263,7 +1263,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of pipeline runs executed in the selected time range", "fieldConfig": { "defaults": { @@ -1310,10 +1310,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1364,7 +1364,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful pipeline runs / Number of total pipeline runs", "fieldConfig": { "defaults": { @@ -1417,10 +1417,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1471,7 +1471,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1584,10 +1584,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1639,7 +1639,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1772,10 +1772,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1826,7 +1826,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -1913,10 +1913,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1967,7 +1967,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful pipeline runs / Number of total pipeline runs", "fieldConfig": { "defaults": { @@ -2014,10 +2014,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2068,7 +2068,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2168,10 +2168,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2243,7 +2243,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2277,7 +2277,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos where id like 'bitbucket%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/CircleCI.json b/grafana/dashboards/mysql/circle-ci.json similarity index 95% rename from grafana/dashboards/CircleCI.json rename to grafana/dashboards/mysql/circle-ci.json index 439d8058de7..c86634eae92 100644 --- a/grafana/dashboards/CircleCI.json +++ b/grafana/dashboards/mysql/circle-ci.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from CircleCI, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate).\n- Data Source Required: CircleCI", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -65,7 +65,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of builds executed in the selected time range", "fieldConfig": { "defaults": { @@ -113,10 +113,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -167,7 +167,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful workflows / Number of total workflows", "fieldConfig": { "defaults": { @@ -218,10 +218,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -272,7 +272,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -406,10 +406,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -460,7 +460,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful workflows / Number of total workflows", "fieldConfig": { "defaults": { @@ -509,10 +509,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -563,7 +563,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -648,10 +648,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -703,7 +703,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean workflows success rate over time.\n2. The workflows being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -808,10 +808,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -862,7 +862,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -978,10 +978,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1033,7 +1033,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1134,10 +1134,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1209,7 +1209,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1241,7 +1241,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from cicd_scopes where id like \"%circleci%\" ", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/ComponentAndFileLevelMetrics.json b/grafana/dashboards/mysql/component-and-file-level-metrics.json similarity index 96% rename from grafana/dashboards/ComponentAndFileLevelMetrics.json rename to grafana/dashboards/mysql/component-and-file-level-metrics.json index 2b369eb8b9f..045e37b4fdb 100644 --- a/grafana/dashboards/ComponentAndFileLevelMetrics.json +++ b/grafana/dashboards/mysql/component-and-file-level-metrics.json @@ -128,10 +128,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -243,10 +243,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -385,10 +385,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -518,10 +518,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -678,10 +678,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -779,10 +779,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -834,7 +834,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", "fieldConfig": { "defaults": { @@ -910,10 +910,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1044,10 +1044,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1125,7 +1125,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", "fieldConfig": { "defaults": { @@ -1201,10 +1201,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1256,7 +1256,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", "fieldConfig": { "defaults": { @@ -1306,10 +1306,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1378,7 +1378,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/ContributorExperience.json b/grafana/dashboards/mysql/contributor-experience.json similarity index 94% rename from grafana/dashboards/ContributorExperience.json rename to grafana/dashboards/mysql/contributor-experience.json index 6ec29831d71..67c681cc73d 100644 --- a/grafana/dashboards/ContributorExperience.json +++ b/grafana/dashboards/mysql/contributor-experience.json @@ -44,7 +44,7 @@ "content": "- Use Cases: This dashboard answers the question \"What makes a great developer experience? And how can we define and track that?\". This dashboard heavily focuses on actionability. All metrics can be deterministically improved as long as OSS maintainers invested time into them.\n- Data Source Required: GitHub", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -59,7 +59,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -108,10 +108,10 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -145,7 +145,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -192,10 +192,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -229,7 +229,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -276,10 +276,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -313,7 +313,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -359,10 +359,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -414,7 +414,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -461,10 +461,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -498,7 +498,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -545,10 +545,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -582,7 +582,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -629,10 +629,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -684,7 +684,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -737,10 +737,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -812,7 +812,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -840,7 +840,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '-', id) from repos", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/DemoAverageRequirementLeadTimeByAssignee.json b/grafana/dashboards/mysql/demo-average-requirement-lead-time-by-assignee.json similarity index 97% rename from grafana/dashboards/DemoAverageRequirementLeadTimeByAssignee.json rename to grafana/dashboards/mysql/demo-average-requirement-lead-time-by-assignee.json index c2c7c1eef4b..66f18b1bc43 100644 --- a/grafana/dashboards/DemoAverageRequirementLeadTimeByAssignee.json +++ b/grafana/dashboards/mysql/demo-average-requirement-lead-time-by-assignee.json @@ -44,7 +44,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Compare the average time spent in each state of a requirement in the last 2 months.\n2. The requirements being calculated are the requirements delivered in the last 2 months.", "fieldConfig": { "defaults": { @@ -103,7 +103,7 @@ "mode": "multi" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -139,7 +139,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -174,7 +174,7 @@ "options": { "showHeader": true }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -221,7 +221,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/DemoCommitCountByAuthor.json b/grafana/dashboards/mysql/demo-commit-count-by-author.json similarity index 97% rename from grafana/dashboards/DemoCommitCountByAuthor.json rename to grafana/dashboards/mysql/demo-commit-count-by-author.json index 7e28acce231..9e3c9f48d39 100644 --- a/grafana/dashboards/DemoCommitCountByAuthor.json +++ b/grafana/dashboards/mysql/demo-commit-count-by-author.json @@ -44,7 +44,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "The number of commits from different commit authors.", "fieldConfig": { "defaults": { @@ -105,7 +105,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -142,7 +142,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -180,7 +180,7 @@ "options": { "showHeader": true }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -228,7 +228,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/DemoDetailedBugInfo.json b/grafana/dashboards/mysql/demo-detailed-bug-info.json similarity index 96% rename from grafana/dashboards/DemoDetailedBugInfo.json rename to grafana/dashboards/mysql/demo-detailed-bug-info.json index 61398ff4b44..80b6c636464 100644 --- a/grafana/dashboards/DemoDetailedBugInfo.json +++ b/grafana/dashboards/mysql/demo-detailed-bug-info.json @@ -44,7 +44,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -106,7 +106,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -140,7 +140,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -241,7 +241,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -286,7 +286,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/DemoHomepage.json b/grafana/dashboards/mysql/demo-homepage.json similarity index 98% rename from grafana/dashboards/DemoHomepage.json rename to grafana/dashboards/mysql/demo-homepage.json index f5947ba1901..9624d8c48b6 100644 --- a/grafana/dashboards/DemoHomepage.json +++ b/grafana/dashboards/mysql/demo-homepage.json @@ -34,7 +34,7 @@ "content": "\n
\n

How fast do we respond to customer requirements?

\n
\n
", "mode": "html" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "time_series", @@ -85,7 +85,7 @@ "content": "\n
\n

Was our quality improved or not?

\n
\n
", "mode": "html" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "time_series", @@ -136,7 +136,7 @@ "content": "\n
\n

Is this month more productive than last?

\n
\n
", "mode": "html" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "time_series", diff --git a/grafana/dashboards/DemoHowFastDoWeRespondToCustomerRequirements.json b/grafana/dashboards/mysql/demo-how-fast-do-we-respond-to-customer-requirements.json similarity index 56% rename from grafana/dashboards/DemoHowFastDoWeRespondToCustomerRequirements.json rename to grafana/dashboards/mysql/demo-how-fast-do-we-respond-to-customer-requirements.json index 5a4b1ccfadb..9dd0d04ac6d 100644 --- a/grafana/dashboards/DemoHowFastDoWeRespondToCustomerRequirements.json +++ b/grafana/dashboards/mysql/demo-how-fast-do-we-respond-to-customer-requirements.json @@ -32,7 +32,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -101,7 +101,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -150,7 +150,7 @@ "content": "
\n\n[Drill down by assignee](/d/q27fk7cnk/demo-average-requirement-lead-time-by-assignee?orgId=1)\n ", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -169,10 +169,10 @@ }, "id": 101, "options": { - "content": "
\n \"Merico\"\n

MARI Guide - Requirement Lead Time

\n
\n\nSection | Description\n:----------------- | :-------------\nMetric Definition | Total duration of requirements from proposal to delivery. It can be divided by flow status in the practice domain or project management system to count the time share of each phase and help locate the links that drag out the requirement delivery cycle.\nMetric Value | The Requirement Lead Time reflects the rapid responsiveness of the R&D team.
In theory, the faster you can deliver value to customers, the better, but other aspects such as whether the delivered value meets customer expectations, requirement throughput, and delivery quality must be considered together. Fast delivery does not necessarily equate to good R&D practices.\n\n***\n#### *M (Measure)*\n1. Count the average or 80th percentile requiremnt lead time for different times.\n2. Counts the average or 80th percentile requiremnt lead time for different projects.
\n3. Count the length of time that requirements stay in different practice domains (requirements analysis, design, development, testing, release) or in different states.\n\n##### *A (Analyze)*\n1. Compare the requirement delivery speed of different projects to find the fastest and slowest delivering projects.\n2. Analyze the trend of the average requirement lead time within each cycle, make a vertical comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.\n3. Analyze the trend of the delivery cycle of 80% of the requirements within each cycle, make a longitudinal comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.

\nWhy choose the 80% quantile instead of using the average?
\nThe point of statistics is to make predictions with real and valid data to support better decisions, while the mean and median cannot have the role of supporting predictions.
\nTypically, the mean and 80% quantile statistics will appear twice as far apart, and the 80% and 99% quantile tend to be approximately twice as related.
\nTherefore, the 80% quantile is a good balance point for prediction.\n
\n4. Analysis compares the length of time requirement stays in different practice domains or different states to identify the most time-consuming links and find the key bottlenecks that affect overall delivery speed.\n5. Requirement lead time is correlated with requirement throughput to identify whether the requirement delivery trend is healthy or not.\n - Healthy trend: requirement lead time is shortened and requirement throughput is increased.\n - Unhealthy trend: longer requirement lead time and lower requirement throughput.\n\n\n##### *R (Review)*\nBased on the analysis results, focus on a few key points and use Ishikawa diagram (fishbone diagram) or Root Cause Analysis (RCA) to conduct root cause analysis, research and review. For example, if the requirement delivery cycle becomes longer in several consecutive statistical cycles, it is necessary to further investigate the length of stay of requirements in different phases and find the longest phase for root cause analysis.\n\n1. The requirements phase takes too long: unclear requirements, frequent changes, overly granular requirements, requirements priorities not clearly defined, insufficient resources or experience of requirements analysts or product managers?\n2. The design phase takes too long: unclear requirement documents, insufficient resources or experience of R&D leaders or architects?\n3. The development phase takes too long: unclear design documents, uneven task distribution, high stream load (parallel tasks), too much technical debt, too many bugs, insufficient resources or experience of developers?\n4. The testing phase takes too long: unclear requirements documentation, poor code quality, few automated tests, insufficient resources or experience of testers?\n5. The release phase takes too long: too long build or deployment time, insufficient resources or experience of operation and maintenance staff?\n\n##### *I (Improve)*\nBased on the review results, focus on the key root causes, and give targeted improvement measures in terms of norms, processes, tools and behaviors, etc., with clear improvement targets, improvement measures, verification cycles and responsible persons.\n\nThe following are the improvement ideas for reference:\n\n1. Communicate with customers or business parties to clarify requirements, reasonably disassemble requirements and define priorities, and invite business parties, R&D leaders and testing leaders to review requirements.\n2. Invite product managers, R&D personnel, and test leaders to conduct design reviews.\n3. Reduce requirement or task granularity, distribute tasks evenly, reduce flow load, increase unit testing, and solve technical debt and code problems in time to reduce the number of bugs and rework.\n4. Test left, develop self-test, code review, increase automated testing, and improve continuous integration capabilities.\n5. Automate deployment, shorten build time, and improve continuous delivery.\n6. Reasonable resource allocation and necessary training for each job holder.\n7. The improvement results should also be quantifiable to facilitate continuous metrics and tracking of improvement effects.", + "content": "
\n \"Merico\"\n

MARI Guide - Requirement Lead Time

\n
\n\nSection | Description\n:----------------- | :-------------\nMetric Definition | Total duration of requirements from proposal to delivery. It can be divided by flow status in the practice domain or project management system to count the time share of each phase and help locate the links that drag out the requirement delivery cycle.\nMetric Value | The Requirement Lead Time reflects the rapid responsiveness of the R&D team.
In theory, the faster you can deliver value to customers, the better, but other aspects such as whether the delivered value meets customer expectations, requirement throughput, and delivery quality must be considered together. Fast delivery does not necessarily equate to good R&D practices.\n\n***\n#### *M (Measure)*\n1. Count the average or 80th percentile requirement lead time for different times.\n2. Counts the average or 80th percentile requirement lead time for different projects.
\n3. Count the length of time that requirements stay in different practice domains (requirements analysis, design, development, testing, release) or in different states.\n\n##### *A (Analyze)*\n1. Compare the requirement delivery speed of different projects to find the fastest and slowest delivering projects.\n2. Analyze the trend of the average requirement lead time within each cycle, make a vertical comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.\n3. Analyze the trend of the delivery cycle of 80% of the requirements within each cycle, make a longitudinal comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.

\nWhy choose the 80% quantile instead of using the average?
\nThe point of statistics is to make predictions with real and valid data to support better decisions, while the mean and median cannot have the role of supporting predictions.
\nTypically, the mean and 80% quantile statistics will appear twice as far apart, and the 80% and 99% quantile tend to be approximately twice as related.
\nTherefore, the 80% quantile is a good balance point for prediction.\n
\n4. Analysis compares the length of time requirement stays in different practice domains or different states to identify the most time-consuming links and find the key bottlenecks that affect overall delivery speed.\n5. Requirement lead time is correlated with requirement throughput to identify whether the requirement delivery trend is healthy or not.\n - Healthy trend: requirement lead time is shortened and requirement throughput is increased.\n - Unhealthy trend: longer requirement lead time and lower requirement throughput.\n\n\n##### *R (Review)*\nBased on the analysis results, focus on a few key points and use Ishikawa diagram (fishbone diagram) or Root Cause Analysis (RCA) to conduct root cause analysis, research and review. For example, if the requirement delivery cycle becomes longer in several consecutive statistical cycles, it is necessary to further investigate the length of stay of requirements in different phases and find the longest phase for root cause analysis.\n\n1. The requirements phase takes too long: unclear requirements, frequent changes, overly granular requirements, requirements priorities not clearly defined, insufficient resources or experience of requirements analysts or product managers?\n2. The design phase takes too long: unclear requirement documents, insufficient resources or experience of R&D leaders or architects?\n3. The development phase takes too long: unclear design documents, uneven task distribution, high stream load (parallel tasks), too much technical debt, too many bugs, insufficient resources or experience of developers?\n4. The testing phase takes too long: unclear requirements documentation, poor code quality, few automated tests, insufficient resources or experience of testers?\n5. The release phase takes too long: too long build or deployment time, insufficient resources or experience of operation and maintenance staff?\n\n##### *I (Improve)*\nBased on the review results, focus on the key root causes, and give targeted improvement measures in terms of norms, processes, tools and behaviors, etc., with clear improvement targets, improvement measures, verification cycles and responsible persons.\n\nThe following are the improvement ideas for reference:\n\n1. Communicate with customers or business parties to clarify requirements, reasonably disassemble requirements and define priorities, and invite business parties, R&D leaders and testing leaders to review requirements.\n2. Invite product managers, R&D personnel, and test leaders to conduct design reviews.\n3. Reduce requirement or task granularity, distribute tasks evenly, reduce flow load, increase unit testing, and solve technical debt and code problems in time to reduce the number of bugs and rework.\n4. Test left, develop self-test, code review, increase automated testing, and improve continuous integration capabilities.\n5. Automate deployment, shorten build time, and improve continuous delivery.\n6. Reasonable resource allocation and necessary training for each job holder.\n7. The improvement results should also be quantifiable to facilitate continuous metrics and tracking of improvement effects.", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -194,7 +194,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/DemoIsThisMonthMoreProductiveThanLast.json b/grafana/dashboards/mysql/demo-is-this-month-more-productive-than-last.json similarity index 97% rename from grafana/dashboards/DemoIsThisMonthMoreProductiveThanLast.json rename to grafana/dashboards/mysql/demo-is-this-month-more-productive-than-last.json index e59181e73be..d59fbc7fce4 100644 --- a/grafana/dashboards/DemoIsThisMonthMoreProductiveThanLast.json +++ b/grafana/dashboards/mysql/demo-is-this-month-more-productive-than-last.json @@ -32,7 +32,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -131,7 +131,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -180,7 +180,7 @@ "content": "
\n\n[Drill down by commit author](/d/F0iYknc7z/demo-commit-count-by-author?orgId=1)\n ", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -202,7 +202,7 @@ "content": "
\n \"Merico\"\n

MARI Guide - Commit Count

\n
\n\nSection | Description\n:------------------ | :-------------\nMetric Definition | The total number of commits.\nMetric Value | The number of commits reflects the frequency of code commits and encourages multiple small step commits per day. While paying attention to the number of times, you should also observe the number of lines of code per commit.\n\n***\n\n#### *M (Measure)*\nThe number of commits can be counted in time, project, team or individual dimensions, and the above 4 dimensions can be cross-tabulated, e.g., the number of commits per week for a project, the average number of commits per person per day for a team.\n\n##### *A (Analyze)*\n1. By project dimension, analyze and compare the number of code commits of different projects horizontally, and locate the projects with the maximum, minimum and lower than average values for research.\n2. By team dimension, analyze and compare the number of code commits of different teams horizontally, and locate the teams with the maximum, minimum, and lower than average values for research.\n3. By individual dimension, analyze and compare the number of code commits of different personnel horizontally, and locate the individuals with the maximum, minimum, and lower than average values for research.\n4. Analyze the trend of the number of code commits in each cycle, compare vertically, and locate the maximum, minimum, and continuously increasing or decreasing cycles for research.\n\n##### *R (Review)*\nBased on the results of metrics analysis, use Ishikawa diagram (fishbone diagram) or root cause analysis (RCA) to analyze, investigate and review the root causes of projects/teams/individuals with low (e.g. lower than average) code submission times in a cycle:\n1. too many complicated requirements with large granularity (estimated workload)?\n2. uneven and full task distribution?\n3. no specification for small step submission?\n\n##### *I (Improve)*\nBased on the review results, focus on the key root causes, give targeted improvement measures in terms of specification, process, tools, behavior, etc., and clarify the improvement target, improvement measures, verification cycle and responsible person.\nTo address the root causes of low code submission in the cycle, the following improvement measures were taken: 1.\n1. reduce the granularity of requirements.\n2. distribute tasks evenly so that each person has a full workload each day.\n3. set small step commit specifications, such as an average number of commits per person per day greater than 1, an average number of lines per commit less than 100, etc.", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -224,7 +224,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/DemoWasOurQualityImprovedOrNot.json b/grafana/dashboards/mysql/demo-was-our-quality-improved-or-not.json similarity index 98% rename from grafana/dashboards/DemoWasOurQualityImprovedOrNot.json rename to grafana/dashboards/mysql/demo-was-our-quality-improved-or-not.json index a44614f495f..3c61266242c 100644 --- a/grafana/dashboards/DemoWasOurQualityImprovedOrNot.json +++ b/grafana/dashboards/mysql/demo-was-our-quality-improved-or-not.json @@ -32,7 +32,7 @@ ], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -101,7 +101,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -147,7 +147,7 @@ "content": "
\n\n[See Detailed Bug Info](/d/s48Lzn5nz/demo-detailed-bug-info?orgId=1)\n ", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -169,7 +169,7 @@ "content": "
\n \"Merico\"\n

MARI Guide - Bug Count per 1k Lines of Code

\n
\n\nSection | Description\n:----------------- | :-------------\nMetric Definition | The ratio of the number of bugs found to the corresponding amount of code or code changes, to characterize the density of overall bugs, including bugs found in testing and online. For example, bug count per 1k lines of code, bug count per 1k code equivalent.\nMetric Value | The bug rate, as a quality indicator, represents the density of bugs, and is one of the important indicators used to assess the quality of software products and testing quality. Usually, the cost of fixing detected bugs is higher in the later stage of the software development life cycle, and this metric is valuable for analyzing and evaluating both online quality and bug fixing cost.\n\n***\n#### *M (Measure)*\n1. Bug count per 1k lines of code by project.\n2. Trends in 'Bug count per 1k lines of code' over time.\n3. Measure historical data to establish year-over-year and historical baseline reference values for 'Bug count per 1k lines of code'.\n\n##### *A (Analyze)*\n1. Year-over-year analysis: The bug rate of similar projects in the same period is compared and analyzed, and the improvement effect of product quality is observed through the rise and fall of the data.\n2. Circumferential analysis: Analyze the bug rate of projects in the recent year, analyze the change of online bug rate according to the time axis, and compare with the historical baseline at the same time to give a judgment analysis of the rise and fall of indicators.\n3. Trend analysis: analyze the trend of bug rate over time (days, weeks, months), judge the trend rise, and evaluate whether the stable cycle of product quality is reasonable by observing changes such as trend slowing-down and smoothing.\n4. Horizontal analysis: Compare the bug rate of multiple projects as a reference to evaluate the quality of software products online.\n5. Classification analysis: Classify and analyze the types of bugs, severity levels, and modules they belong to, and identify the key issues that show aggregated distribution.\n\n##### *R (Review)*\nFor the high severity level of online bugs should be a complete review, according to the timeline, role dimensions, the sequence of events on the root cause of bugs to dig, locate the key issues.\nAccording to the quantitative conclusions drawn from the analysis, further data drilling and root cause mining can be organized for bugs in several dimensions, including whether they are missed, the module they belong to, the cause, the occurrence cycle, and the resolution.\n1. bug escape rate: derived from [number of online bugs / (number of online bugs + number of bugs found in test)], this indicator can be compared with historical data, if the data exceeds the acceptable range of history and testing department, then it is necessary to conduct leak analysis. If the data exceeds the acceptable interval of history and testing department, it is necessary to analyze the missed test and confirm whether the use case is missed or not covered, so as to strengthen the use case design and management.\n2. defective module: The defective module can be located to the key module where the problem is concentrated, and targeted improvement measures are required for each link from requirements, design, development to testing, and typical problems are located for the bugive module to establish targeted measures.\n3. bug generation causes: through the cause analysis of bugs, similar bugs bugs can be put together, so that bugs belonging to the same category and accounting for a high percentage of bugs will be highlighted, so that it is easy to take out the bugs with more concentrated causes and jointly discuss the next improvement measures to precisely reduce the number of similar bugs.\n4. bug occurrence cycle: analyze the bug occurrence cycle, determine whether the users use the system frequently, whether the system has been updated or optimized, whether the system has been refactored, etc., which will cause a long and short bug occurrence cycle after the launch, through the analysis of the bug cycle length, to draw some valuable conclusions about the stability of the system.\n5. bug resolution: statistics on the resolution of bugs, which bugs are not reproduced, which are temporarily handled, and which are in need of continuous improvement. For the temporary resolution of bugs, analyze whether it will cause another bug elsewhere in the system, whether the user can receive the temporary resolution, what development and testing need to focus on similar bugs, and whether similar bugs need to be tested elsewhere for horizontal expansion. Those that require continuous improvement need to be further tracked by testing until the problem is resolved.\n\n##### *I (Improve)*\nThrough root cause mining, starting from the key bugs, the key problems of each link to locate, in accordance with the principle that the later the bugs are found, the higher the cost and complexity of the solution, in addition to the test design and implementation to start improving (refer to the improvement links of the number of bugs on the line), more should start the construction of quality from the upstream of the software engineering stage, to achieve the forward movement of the bug discovery stage, such as\n1. optimizing static scan rules according to the type, number and severity level of static scan problems to reduce false positives and expose as many serious problems as possible (quality over quantity).\n2. Define the requirement of resolution ratio for different severity level problems to control the backlog of serious problems.\n3. Establish code review system, strategy, and encourage the promotion of code review implementation.\n4. Establish unit test coverage ratio or unit test coverage condition requirements, e.g. functions with circle complexity greater than 10 shall be covered by unit tests.\nImplement improvement measures and clarify the improvement target, improvement measures, verification cycle and responsible person. Do a new round of MARI (Measure, Analysis, Review, and Verification) for the improvement effect to quantify the improvement effect.", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -191,7 +191,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", diff --git a/grafana/dashboards/mysql/developer-productivity-hours.json b/grafana/dashboards/mysql/developer-productivity-hours.json new file mode 100644 index 00000000000..2259f6d5bdd --- /dev/null +++ b/grafana/dashboards/mysql/developer-productivity-hours.json @@ -0,0 +1,162 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Chat and completion events by hour of day", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, + "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(hr AS CHAR), 2, '0') AS 'Hour',\n SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) AS hr, COUNT(*) AS chat, 0 AS comp\n FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) AS hr, 0 AS chat, COUNT(*) AS comp\n FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) t GROUP BY hr ORDER BY hr", + "refId": "A" + } + ], + "title": "AI Activity by Hour of Day (UTC)", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average prompt complexity and response richness by hour", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "showPoints": "auto", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "id": 2, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(HOUR(timestamp) AS CHAR), 2, '0') AS 'Hour',\n ROUND(AVG(prompt_length)) AS 'Avg Prompt Length',\n ROUND(AVG(response_length)) AS 'Avg Response Length'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY HOUR(timestamp)\nORDER BY HOUR(timestamp)", + "refId": "A" + } + ], + "title": "Prompt & Response Length by Hour", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Steering and spec mode usage concentration by hour", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, + "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "id": 3, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(HOUR(timestamp) AS CHAR), 2, '0') AS 'Hour',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) AS 'Plain Chat'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY HOUR(timestamp)\nORDER BY HOUR(timestamp)", + "refId": "A" + } + ], + "title": "Feature Usage by Hour", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Day-of-week activity pattern", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, + "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, + "id": 4, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE DAYOFWEEK(timestamp)\n WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue'\n WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat'\n END AS 'Day',\n SUM(chat) AS 'Chat Events', SUM(comp) AS 'Completions'\nFROM (\n SELECT timestamp, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY timestamp\n UNION ALL\n SELECT timestamp, 0, COUNT(*) FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY timestamp\n) t\nGROUP BY DAYOFWEEK(timestamp)\nORDER BY DAYOFWEEK(timestamp)", + "refId": "A" + } + ], + "title": "AI Activity by Day of Week", + "type": "barchart" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "kiro", "productivity", "hours"], + "templating": { "list": [] }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Developer AI Productivity Hours", + "uid": "kiro_productivity_hours", + "version": 1 +} diff --git a/grafana/dashboards/DORAByTeam.json b/grafana/dashboards/mysql/dora-by-team.json similarity index 97% rename from grafana/dashboards/DORAByTeam.json rename to grafana/dashboards/mysql/dora-by-team.json index 766802ed6f8..03735ab89ed 100644 --- a/grafana/dashboards/DORAByTeam.json +++ b/grafana/dashboards/mysql/dora-by-team.json @@ -49,7 +49,7 @@ "content": "- See [how to config](https://devlake.apache.org/docs/DORA) this dashboard\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- You can validate/debug this dashboard with the [DORA validation dashboard](/grafana/d/KGkUnV-Vz/dora-dashboard-validation) \n- You also need to do [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard. \n- DORA benchmarks vary in different years. You can switch the benchmarks to change them.\n- In DORA's official report in 2023, metric 'failed deployment recovery time' has replaced 'MTTR'.\n- How does this work? \n - Gets the author of the specific commit and then navigates to the team the user belongs to. \n - Gets the team from the PR's author. \n - Gets the team from the commit author.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -63,7 +63,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -198,10 +198,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -231,7 +231,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -315,10 +315,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -370,7 +370,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -454,10 +454,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -487,7 +487,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -571,10 +571,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -604,7 +604,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -689,10 +689,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -722,7 +722,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -799,7 +799,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -829,7 +829,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -901,10 +901,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -934,7 +934,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1025,10 +1025,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -1058,7 +1058,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1135,10 +1135,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -1181,7 +1181,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from teams", "hide": 0, "includeAll": true, @@ -1202,7 +1202,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -1223,7 +1223,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/DORADebug.json b/grafana/dashboards/mysql/dora-debug.json similarity index 95% rename from grafana/dashboards/DORADebug.json rename to grafana/dashboards/mysql/dora-debug.json index 86125bef4f0..59318074fa2 100644 --- a/grafana/dashboards/DORADebug.json +++ b/grafana/dashboards/mysql/dora-debug.json @@ -43,7 +43,7 @@ "content": "This dashboard is designed to validate the [DORA dashboard](/grafana/d/qNo8_0M4z/dora?orgId=1).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -104,7 +104,7 @@ "content": "- See the definition and calculation logic of [Deployment Frequency](https://devlake.apache.org/docs/Metrics/DeploymentFrequency)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n- Transformation Required: Define `deployments` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -118,7 +118,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -189,16 +189,16 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "SELECT\n\tpm.project_name,\n\tIF(pm.project_name in ($project),'This project is selected','Not Selected') as select_status,\n\tIF(cdc._raw_data_table != '', cdc._raw_data_table, cdc.cicd_scope_id) as _raw_data_table,\n\tresult,\n\tenvironment,\n\tcount(distinct cdc.id) as deployment_commit_count, \n\tcount(distinct cdc.cicd_deployment_id) as deployment_count\nFROM cicd_deployment_commits cdc\nLEFT join project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\nWHERE $__timeFilter(cdc.finished_date)\nGROUP BY pm.project_name, select_status, _raw_data_table,result,environment", + "rawSql": "SELECT\n\tpm.project_name,\n\tIF(pm.project_name in ($project),'This project is selected','Not Selected') as select_status,\n\tIF(cdc._raw_data_table != '', cdc._raw_data_table, cdc.cicd_scope_id) as _raw_data_table,\n\tresult,\n\tenvironment,\n\tcount(distinct cdc.id) as deployment_commit_count, \n\tcount(distinct cdc.cicd_deployment_id) as deployment_count\nFROM cicd_deployment_commits cdc\nLEFT join project_mapping pm on cdc.cicd_scope_id = pm.row_id and pm.`table` = 'cicd_scopes'\nWHERE $__timeFilter(cdc.finished_date)\nGROUP BY 1, 2, 3, 4, 5", "refId": "A", "select": [ [ @@ -226,7 +226,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -310,10 +310,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -365,7 +365,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -436,10 +436,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -473,7 +473,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -550,7 +550,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -602,7 +602,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -681,10 +681,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -718,7 +718,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -798,10 +798,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -835,7 +835,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -915,10 +915,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -998,7 +998,7 @@ "content": "- See the definition and calculation logic of [Median Lead Time for Changes](https://devlake.apache.org/docs/Metrics/LeadTimeForChanges)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n- Transformation Required: Define `deployments` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- Validatation Steps below:\n - Step 1 - check the data integrity of PRs in table `pull_requests`\n - Step 2 - check the data integrity of deployment_commit in table `cicd_deployment_commits`\n - Step 3 - check if a deployment_commit is associated with the correct PR in table `project_pr_metrics`\n - Step 4 - check if metrics like PR Coding/Pickup/Review/Deploy/cycle time are correct in table `project_pr_metrics`\n - Step 5 - check if the median lead time for changes in each month is correct", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1012,7 +1012,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1059,10 +1059,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1096,7 +1096,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1179,10 +1179,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1234,7 +1234,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1348,10 +1348,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1385,7 +1385,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1456,10 +1456,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1511,7 +1511,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1565,10 +1565,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1598,7 +1598,7 @@ ] }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1632,7 +1632,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1702,10 +1702,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1739,7 +1739,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1859,10 +1859,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1896,7 +1896,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1958,10 +1958,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1995,7 +1995,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2040,10 +2040,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2073,7 +2073,7 @@ ] }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2107,7 +2107,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2148,10 +2148,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2181,7 +2181,7 @@ ] }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2215,7 +2215,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2256,10 +2256,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2289,7 +2289,7 @@ ] }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2323,7 +2323,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2364,10 +2364,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2397,7 +2397,7 @@ ] }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2431,7 +2431,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2472,10 +2472,10 @@ "sizing": "auto", "text": {} }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2509,7 +2509,7 @@ "type": "gauge" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2608,10 +2608,10 @@ }, "showHeader": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2641,7 +2641,7 @@ ] } ], - "title": "Step 5 - check the median change lead time for each month in Figure 4 (Compare the change_lead_time with the max ranks in GREEN before the first occurence of ORANGE in each month)", + "title": "Step 5 - check the median change lead time for each month in Figure 4 (Compare the change_lead_time with the max ranks in GREEN before the first occurrence of ORANGE in each month)", "type": "table" }, { @@ -2691,7 +2691,7 @@ "content": "- See the definition and calculation logic of [${title_value}](https://devlake.apache.org/docs/Metrics/MTTR)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2725,7 +2725,7 @@ "content": "- See the definition and calculation logic of [Change Failure Rate](https://devlake.apache.org/docs/Metrics/CFR)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2739,7 +2739,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2800,10 +2800,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2837,7 +2837,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2920,10 +2920,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2975,7 +2975,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -3036,10 +3036,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -3093,7 +3093,7 @@ "content": "\n\nIn this case:\n\n- Deployment-1 maps to Incident-1\n- Deployment-3 maps to Incident-2 and Incident-3\n- Deployment-2,4,5 doesn't map to any Incident", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -3108,7 +3108,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -3200,10 +3200,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3255,7 +3255,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -3338,10 +3338,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3393,7 +3393,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -3455,16 +3455,16 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with _deployments as(\n select\n distinct d.cicd_deployment_id as deployment_id,\n d.result,\n d.environment,\n d.finished_date,\n d.cicd_scope_id,\n pm.project_name\n from\n cicd_deployment_commits d\n join project_mapping pm on d.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n where\n -- only result needs to specified, not envioronment\n d.result = 'SUCCESS' -- choose your project_name\n and pm.project_name in ($project)\n),\n_incidents as(\n select\n distinct i.id as issue_id,\n i.created_date,\n pm.project_name\n from\n incidents i\n join project_mapping pm on i.scope_id = pm.row_id\n and i.`table` = pm.`table`\n where\n -- choose your project_name\n pm.project_name in ($project)\n)\nselect\n deployment_id as id,\n 'DEPLOYMENT' as type,\n finished_date as time\nfrom\n _deployments\nunion\nselect\n issue_id as id,\n 'INCIDENT' as type,\n created_date as time\nfrom\n _incidents\norder by\n time", + "rawSql": "with _deployments as(\n select\n distinct d.cicd_deployment_id as deployment_id,\n d.result,\n d.environment,\n d.finished_date,\n d.cicd_scope_id,\n pm.project_name\n from\n cicd_deployment_commits d\n join project_mapping pm on d.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n where\n -- only result needs to specified, not environment\n d.result = 'SUCCESS' -- choose your project_name\n and pm.project_name in ($project)\n),\n_incidents as(\n select\n distinct i.id as issue_id,\n i.created_date,\n pm.project_name\n from\n incidents i\n join project_mapping pm on i.scope_id = pm.row_id\n and i.`table` = pm.`table`\n where\n -- choose your project_name\n pm.project_name in ($project)\n)\nselect\n deployment_id as id,\n 'DEPLOYMENT' as type,\n finished_date as time\nfrom\n _deployments\nunion\nselect\n issue_id as id,\n 'INCIDENT' as type,\n created_date as time\nfrom\n _incidents\norder by\n time", "refId": "A", "select": [ [ @@ -3492,7 +3492,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -3554,10 +3554,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3609,7 +3609,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -3701,10 +3701,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3769,7 +3769,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -3790,7 +3790,7 @@ "text": "", "value": "" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(Url, '--', id) from pull_requests", "hide": 0, "includeAll": false, @@ -3811,7 +3811,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -3832,7 +3832,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/DORADetails-ChangeFailureRate.json b/grafana/dashboards/mysql/dora-details-change-failure-rate.json similarity index 88% rename from grafana/dashboards/DORADetails-ChangeFailureRate.json rename to grafana/dashboards/mysql/dora-details-change-failure-rate.json index 8048b282b1f..878540d0fdb 100644 --- a/grafana/dashboards/DORADetails-ChangeFailureRate.json +++ b/grafana/dashboards/mysql/dora-details-change-failure-rate.json @@ -57,7 +57,7 @@ "content": "This dashboard shows the details about [Change Failure Rate](https://devlake.apache.org/docs/Metrics/CFR) in DORA.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -71,7 +71,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -154,17 +154,17 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n -- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM\n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in (${project})\n and cdc.result = 'SUCCESS'\n and cdc.environment = 'PRODUCTION'\n GROUP BY\n 1\n HAVING\n $__timeFilter(max(cdc.finished_date))\n),\n_failure_caused_by_deployments as (\n -- calculate the number of incidents caused by each deployment\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n count(distinct i.id) as has_incident\n FROM\n _deployments d\n left join project_incident_deployment_relationships pim on d.deployment_id = pim.deployment_id\n left join incidents i on pim.id = i.id\n GROUP BY\n 1,\n 2\n)\nSELECT\n sum(has_incident) / count(deployment_id) as \"change_failure_rate\"\nFROM\n _failure_caused_by_deployments", + "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n -- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM\n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in (${project})\n and cdc.result = 'SUCCESS'\n and cdc.environment = 'PRODUCTION'\n GROUP BY\n 1\n HAVING\n $__timeFilter(max(cdc.finished_date))\n),\n_failure_caused_by_deployments as (\n -- calculate the number of incidents caused by each deployment\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n CASE WHEN count(distinct i.id) > 0 THEN 1 ELSE 0 END as has_incident\n FROM\n _deployments d\n left join project_incident_deployment_relationships pim on d.deployment_id = pim.deployment_id\n left join incidents i on pim.id = i.id\n GROUP BY\n 1,\n 2\n)\nSELECT\n sum(has_incident) / count(deployment_id) as \"change_failure_rate\"\nFROM\n _failure_caused_by_deployments", "refId": "A", "select": [ [ @@ -209,7 +209,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -355,10 +355,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -410,7 +410,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -492,17 +492,17 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n -- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM\n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in (${project})\n and cdc.result = 'SUCCESS'\n and cdc.environment = 'PRODUCTION'\n GROUP BY\n 1\n HAVING\n $__timeFilter(max(cdc.finished_date))\n),\n_failure_caused_by_deployments as (\n -- calculate the number of incidents caused by each deployment\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n count(distinct i.id) as has_incident\n FROM\n _deployments d\n left join project_incident_deployment_relationships pim on d.deployment_id = pim.deployment_id\n left join incidents i on pim.id = i.id\n GROUP BY\n 1,\n 2\n)\nSELECT\n sum(has_incident) as \"incident count\",\n count(deployment_id) as \"deployment count\"\nFROM\n _failure_caused_by_deployments", + "rawSql": "-- Metric 3: change failure rate\nwith _deployments as (\n -- When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date.\n SELECT\n cdc.cicd_deployment_id as deployment_id,\n max(cdc.finished_date) as deployment_finished_date\n FROM\n cicd_deployment_commits cdc\n JOIN project_mapping pm on cdc.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n WHERE\n pm.project_name in (${project})\n and cdc.result = 'SUCCESS'\n and cdc.environment = 'PRODUCTION'\n GROUP BY\n 1\n HAVING\n $__timeFilter(max(cdc.finished_date))\n),\n_failure_caused_by_deployments as (\n -- calculate the number of incidents caused by each deployment\n SELECT\n d.deployment_id,\n d.deployment_finished_date,\n CASE WHEN count(distinct i.id) > 0 THEN 1 ELSE 0 END as has_incident\n FROM\n _deployments d\n left join project_incident_deployment_relationships pim on d.deployment_id = pim.deployment_id\n left join incidents i on pim.id = i.id\n GROUP BY\n 1,\n 2\n)\nSELECT\n sum(has_incident) as \"failed deployment count\",\n count(deployment_id) as \"deployment count\"\nFROM\n _failure_caused_by_deployments", "refId": "A", "select": [ [ @@ -546,7 +546,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -617,17 +617,17 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with _deployments as(\n select\n distinct d.cicd_deployment_id as deployment_id,\n d.result,\n d.environment,\n d.finished_date,\n d.cicd_scope_id,\n pm.project_name\n from\n cicd_deployment_commits d\n join project_mapping pm on d.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n where\n -- only result needs to specified, not envioronment\n d.result = 'SUCCESS' -- choose your project_name\n and pm.project_name in ($project)\n and $__timeFilter(d.finished_date)\n),\n_incidents as(\n select\n distinct i.id as issue_id,\n i.created_date,\n pm.project_name\n from\n incidents i\n join project_mapping pm on i.scope_id = pm.row_id\n and i.`table` = pm.`table`\n where\n -- choose your project_name\n pm.project_name in ($project)\n and $__timeFilter(i.created_date)\n)\nselect\n finished_date as 'Time (Ascending)',\n deployment_id as 'Entity ID',\n 'DEPLOYMENT' as 'Entity Type (Deployment/Incident)'\nfrom\n _deployments\nunion\nselect\n created_date as 'Time (Ascending)',\n issue_id as 'Entity ID',\n 'INCIDENT' as 'Entity Type (Deployment/Incident)'\nfrom\n _incidents\norder by\n 1", + "rawSql": "with _deployments as(\n select\n distinct d.cicd_deployment_id as deployment_id,\n d.result,\n d.environment,\n d.finished_date,\n d.cicd_scope_id,\n pm.project_name\n from\n cicd_deployment_commits d\n join project_mapping pm on d.cicd_scope_id = pm.row_id\n and pm.`table` = 'cicd_scopes'\n where\n -- only result needs to specified, not environment\n d.result = 'SUCCESS' -- choose your project_name\n and pm.project_name in ($project)\n and $__timeFilter(d.finished_date)\n),\n_incidents as(\n select\n distinct i.id as issue_id,\n i.created_date,\n pm.project_name\n from\n incidents i\n join project_mapping pm on i.scope_id = pm.row_id\n and i.`table` = pm.`table`\n where\n -- choose your project_name\n pm.project_name in ($project)\n and $__timeFilter(i.created_date)\n)\nselect\n finished_date as 'Time (Ascending)',\n deployment_id as 'Entity ID',\n 'DEPLOYMENT' as 'Entity Type (Deployment/Incident)'\nfrom\n _deployments\nunion\nselect\n created_date as 'Time (Ascending)',\n issue_id as 'Entity ID',\n 'INCIDENT' as 'Entity Type (Deployment/Incident)'\nfrom\n _incidents\norder by\n 1", "refId": "A", "select": [ [ @@ -686,7 +686,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -707,7 +707,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -728,7 +728,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/DORADetails-DeploymentFrequency.json b/grafana/dashboards/mysql/dora-details-deployment-frequency.json similarity index 97% rename from grafana/dashboards/DORADetails-DeploymentFrequency.json rename to grafana/dashboards/mysql/dora-details-deployment-frequency.json index 44c49c5ad23..3023118a526 100644 --- a/grafana/dashboards/DORADetails-DeploymentFrequency.json +++ b/grafana/dashboards/mysql/dora-details-deployment-frequency.json @@ -56,7 +56,7 @@ "content": "This dashboard shows the details about [deployment frequency](https://devlake.apache.org/docs/Metrics/DeploymentFrequency) in DORA.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -70,7 +70,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -247,10 +247,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -302,7 +302,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -365,10 +365,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -420,7 +420,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -493,10 +493,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -548,7 +548,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -623,10 +623,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -678,7 +678,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -752,10 +752,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -827,12 +827,12 @@ "content": "# 2023 Benchmarks\n- Elite: >= 5 days per week\n- High: >= 1 day per week\n- Medium: >= 1 day per month\n- Low: < 1 day per month\n# 2021 Benchmarks\n- Elite: >= 5 days per week\n- High: >= 1 day per month\n- Medium: >= 1 day per six months\n- Low: < 1 day per six months", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "title": "DORA Report", "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -915,10 +915,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -970,7 +970,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1053,10 +1053,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1108,7 +1108,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1191,10 +1191,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1262,7 +1262,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -1283,7 +1283,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -1304,7 +1304,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json b/grafana/dashboards/mysql/dora-details-failed-deployment-recovery-time.json similarity index 96% rename from grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json rename to grafana/dashboards/mysql/dora-details-failed-deployment-recovery-time.json index 4dfd6fb0638..aabdd461e4c 100644 --- a/grafana/dashboards/DORADetails-FailedDeploymentRecoveryTime.json +++ b/grafana/dashboards/mysql/dora-details-failed-deployment-recovery-time.json @@ -56,7 +56,7 @@ "content": "This dashboard shows the details about [Failed Deployment Recovery Time\n](https://devlake.apache.org/docs/Metrics/MTTR) in DORA.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -70,7 +70,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -113,10 +113,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -146,7 +146,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -304,10 +304,10 @@ } ] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -359,7 +359,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -401,10 +401,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -447,7 +447,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/DORADetails-LeadTimeforChanges.json b/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json similarity index 95% rename from grafana/dashboards/DORADetails-LeadTimeforChanges.json rename to grafana/dashboards/mysql/dora-details-lead-timefor-changes.json index a68bd1a246e..d6464b4207d 100644 --- a/grafana/dashboards/DORADetails-LeadTimeforChanges.json +++ b/grafana/dashboards/mysql/dora-details-lead-timefor-changes.json @@ -55,7 +55,7 @@ "content": "- This dashboard shows the details about [Lead Time for Changes](https://devlake.apache.org/docs/Metrics/LeadTimeForChanges) in DORA.\n- It displays the statistics for pull requests **deployed** within the selected time range.", "mode": "markdown" }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -69,7 +69,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -122,10 +122,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -177,7 +177,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -230,10 +230,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -285,7 +285,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -369,10 +369,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -424,7 +424,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -508,10 +508,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -563,7 +563,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -647,10 +647,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -702,7 +702,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Lead time for changes = median(PR deployed date - PR's first commit's authored date)", "fieldConfig": { "defaults": { @@ -823,10 +823,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -895,7 +895,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -916,7 +916,7 @@ "text": "2023", "value": "2023" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select dora_report from dora_benchmarks", "hide": 0, "includeAll": false, @@ -937,7 +937,7 @@ "text": "Failed Deployment Recovery Time", "value": "Failed Deployment Recovery Time" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT \n CASE \n WHEN dora_report = '2023' THEN \"Failed Deployment Recovery Time\"\n WHEN dora_report = '2021' THEN \"Median Time to Restore Service\"\n ELSE NULL \n END AS title_value\nFROM dora_benchmarks\nWHERE dora_report = '${dora_report:raw}'", "hide": 2, "includeAll": false, diff --git a/grafana/dashboards/DORADetails-TimetoRestoreService.json b/grafana/dashboards/mysql/dora-details-timeto-restore-service.json similarity index 95% rename from grafana/dashboards/DORADetails-TimetoRestoreService.json rename to grafana/dashboards/mysql/dora-details-timeto-restore-service.json index 24f99a534a1..127ec734ed9 100644 --- a/grafana/dashboards/DORADetails-TimetoRestoreService.json +++ b/grafana/dashboards/mysql/dora-details-timeto-restore-service.json @@ -57,7 +57,7 @@ "content": "This dashboard shows the details about [Time to restore service\n](https://devlake.apache.org/docs/Metrics/MTTR) in DORA.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -71,7 +71,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -113,10 +113,10 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -146,7 +146,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -291,10 +291,10 @@ } ] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -347,7 +347,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -388,10 +388,10 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "hide": false, @@ -435,7 +435,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/EngineeringOverview.json b/grafana/dashboards/mysql/engineering-overview.json similarity index 94% rename from grafana/dashboards/EngineeringOverview.json rename to grafana/dashboards/mysql/engineering-overview.json index e2e6cdede52..ad6d5dfefb6 100644 --- a/grafana/dashboards/EngineeringOverview.json +++ b/grafana/dashboards/mysql/engineering-overview.json @@ -40,10 +40,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard is to overview the Git and project management metrics.\n- Data Source Required: Jira + GitHub, or Jira + GitLab.", + "content": "- Use Cases: This dashboard is to overview the Git and project management metrics.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira)", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -58,7 +58,7 @@ "type": "text" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -109,10 +109,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -164,7 +164,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -243,10 +243,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -298,7 +298,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -351,10 +351,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -405,7 +405,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -494,10 +494,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -530,7 +530,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -581,10 +581,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -636,7 +636,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -715,10 +715,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -770,7 +770,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -830,10 +830,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -885,7 +885,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -971,10 +971,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1026,7 +1026,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1080,10 +1080,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1135,7 +1135,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1221,10 +1221,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1276,7 +1276,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1334,10 +1334,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1389,7 +1389,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1473,10 +1473,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1528,7 +1528,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1575,10 +1575,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1630,7 +1630,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1707,10 +1707,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1762,7 +1762,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1818,10 +1818,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1873,7 +1873,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1959,10 +1959,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2014,7 +2014,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2068,7 +2068,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2120,7 +2120,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2201,7 +2201,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -2255,7 +2255,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2287,7 +2287,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -2308,7 +2308,7 @@ "text": "", "value": "" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select\n distinct(concat(date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m') , ':', date_format(DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY), '%Y-%m-%d'))) as month\nfrom\n issues i\norder by month desc", "hide": 0, "includeAll": false, @@ -2333,7 +2333,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct priority from issues", "description": "Customize what prioriti(es) are considered \"critical\"", "hide": 0, @@ -2355,7 +2355,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/EngineeringThroughputAndCycleTimeTeamView.json b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time-team-view.json similarity index 96% rename from grafana/dashboards/EngineeringThroughputAndCycleTimeTeamView.json rename to grafana/dashboards/mysql/engineering-throughput-and-cycle-time-team-view.json index 72c2e45f72a..86d09e2afee 100644 --- a/grafana/dashboards/EngineeringThroughputAndCycleTimeTeamView.json +++ b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time-team-view.json @@ -40,10 +40,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Source Required: GitHub and Jira([transformation](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is). You also need to do [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard.", + "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira) ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is)\n- You also need to do [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard.", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -84,7 +84,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -200,10 +200,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -255,7 +255,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -371,10 +371,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -408,7 +408,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -524,10 +524,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -561,7 +561,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -677,10 +677,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -740,7 +740,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -856,10 +856,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -911,7 +911,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1027,10 +1027,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1108,7 +1108,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1224,10 +1224,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1279,7 +1279,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1395,10 +1395,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1476,7 +1476,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1592,10 +1592,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1647,7 +1647,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1763,10 +1763,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1844,7 +1844,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1960,10 +1960,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -2015,7 +2015,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2131,10 +2131,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -2212,7 +2212,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2325,10 +2325,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2380,7 +2380,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2492,10 +2492,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2547,7 +2547,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2659,10 +2659,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2714,7 +2714,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2826,10 +2826,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2881,7 +2881,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2993,10 +2993,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3074,7 +3074,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -3186,10 +3186,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3261,7 +3261,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -3288,7 +3288,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -3311,7 +3311,7 @@ "text": "None", "value": "" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from teams", "hide": 0, "includeAll": false, @@ -3334,7 +3334,7 @@ "text": "None", "value": "" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from teams", "hide": 0, "includeAll": false, @@ -3360,7 +3360,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct priority from issues where priority != ''", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/EngineeringThroughputAndCycleTime.json b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json similarity index 94% rename from grafana/dashboards/EngineeringThroughputAndCycleTime.json rename to grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json index 718f656ee5a..958edd3a563 100644 --- a/grafana/dashboards/EngineeringThroughputAndCycleTime.json +++ b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json @@ -39,10 +39,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Source Required: GitHub and Jira([transformation](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is)", + "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira) ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is)", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -83,7 +83,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -199,10 +199,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -236,7 +236,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -352,7 +352,7 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -388,7 +388,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -473,10 +473,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -528,7 +528,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -613,10 +613,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "hide": false, @@ -650,7 +650,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -736,10 +736,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -773,7 +773,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -858,10 +858,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -913,7 +913,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -995,10 +995,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1058,7 +1058,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1139,10 +1139,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1194,7 +1194,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1275,10 +1275,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1330,7 +1330,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1442,10 +1442,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1497,7 +1497,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1609,10 +1609,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1664,7 +1664,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1776,10 +1776,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1851,7 +1851,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1879,7 +1879,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct name from projects", "hide": 0, "includeAll": true, @@ -1905,7 +1905,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct priority from issues where priority != ''", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/GitHub.json b/grafana/dashboards/mysql/git-hub.json similarity index 95% rename from grafana/dashboards/GitHub.json rename to grafana/dashboards/mysql/git-hub.json index 89bc8842b50..c9df4e68504 100644 --- a/grafana/dashboards/GitHub.json +++ b/grafana/dashboards/mysql/git-hub.json @@ -50,7 +50,7 @@ "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from GitHub.\n- Data Source Required: GitHub", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -91,7 +91,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -140,10 +140,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -195,7 +195,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -276,10 +276,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -357,7 +357,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -406,10 +406,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -443,7 +443,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -541,10 +541,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -596,7 +596,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -650,10 +650,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -687,7 +687,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -771,10 +771,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -826,7 +826,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -906,10 +906,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -961,7 +961,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1094,10 +1094,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1157,7 +1157,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1206,10 +1206,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1261,7 +1261,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1356,10 +1356,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1411,7 +1411,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1495,10 +1495,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1576,7 +1576,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1628,10 +1628,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1683,7 +1683,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1807,10 +1807,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1862,7 +1862,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1911,10 +1911,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1966,7 +1966,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2046,10 +2046,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -2101,7 +2101,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2155,10 +2155,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2192,7 +2192,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2275,10 +2275,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2330,7 +2330,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2382,10 +2382,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -2419,7 +2419,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2502,10 +2502,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2570,7 +2570,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of workflow runs executed in the selected time range", "fieldConfig": { "defaults": { @@ -2617,10 +2617,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2671,7 +2671,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful workflow runs / Number of total workflow runs", "fieldConfig": { "defaults": { @@ -2724,10 +2724,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2778,7 +2778,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2891,10 +2891,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2946,7 +2946,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful workflow runs / Number of total workflow runs", "fieldConfig": { "defaults": { @@ -3079,10 +3079,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3133,7 +3133,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Mean Workflow Runs success rate over time.\n2. The workflow runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -3236,10 +3236,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3290,7 +3290,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of successful workflow runs / Number of total workflow runs", "fieldConfig": { "defaults": { @@ -3337,10 +3337,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3391,7 +3391,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -3491,10 +3491,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -3566,7 +3566,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -3600,7 +3600,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos where id like 'github%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/mysql/github-copilot-adoption-by-team-and-user.json b/grafana/dashboards/mysql/github-copilot-adoption-by-team-and-user.json new file mode 100644 index 00000000000..af695feed12 --- /dev/null +++ b/grafana/dashboards/mysql/github-copilot-adoption-by-team-and-user.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "mysql", + "description": "Per-team and per-user Copilot activity in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.team_slug as \"Team\", udm.user_login as \"User\", SUM(udm.user_initiated_interaction_count) as \"Interactions\", SUM(udm.code_generation_activity_count) as \"Suggestions\", SUM(udm.code_acceptance_activity_count) as \"Acceptances\", ROUND(SUM(udm.code_acceptance_activity_count) * 100.0 / NULLIF(SUM(udm.code_generation_activity_count), 0), 1) as \"Acceptance Rate %\" FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.team_slug, udm.user_login HAVING SUM(udm.code_generation_activity_count) > 0 ORDER BY \"Suggestions\" DESC LIMIT 50", + "refId": "A" + } + ], + "title": "Team and User Breakdown", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Weekly suggestions by team over the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT DATE(DATE_SUB(ut.day, INTERVAL WEEKDAY(ut.day) DAY)) as time, ut.team_slug as metric, SUM(udm.code_generation_activity_count) as value FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY time, ut.team_slug ORDER BY time", + "refId": "A" + } + ], + "title": "Top Teams by Weekly Suggestions", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "copilot", + "team", + "gh-copilot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "mysql", + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "mysql", + "definition": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Copilot Adoption by Team and User", + "uid": "github-copilot-adoption-by-team-and-user", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/mysql/github-copilot-adoption-by-team.json b/grafana/dashboards/mysql/github-copilot-adoption-by-team.json new file mode 100644 index 00000000000..34095e8ece9 --- /dev/null +++ b/grafana/dashboards/mysql/github-copilot-adoption-by-team.json @@ -0,0 +1,330 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "mysql", + "description": "Distinct teams with Copilot activity in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT team_slug) as \"Active Teams\" FROM _tool_copilot_user_teams WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Active Teams", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Distinct users mapped to teams in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) as \"Active Users\" FROM _tool_copilot_user_teams WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Active Users in Teams", + "type": "stat" + }, + { + "datasource": "mysql", + "description": "Teams ranked by Copilot suggestions generated in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.team_slug as \"Team\", COUNT(DISTINCT ut.user_id) as \"Users\", SUM(udm.code_generation_activity_count) as \"Suggestions\", SUM(udm.code_acceptance_activity_count) as \"Acceptances\", ROUND(SUM(udm.code_acceptance_activity_count) * 100.0 / NULLIF(SUM(udm.code_generation_activity_count), 0), 1) as \"Acceptance Rate %\" FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.team_slug HAVING SUM(udm.code_generation_activity_count) > 0 ORDER BY \"Suggestions\" DESC LIMIT 15", + "refId": "A" + } + ], + "title": "Top Teams by Suggestions", + "type": "table" + }, + { + "datasource": "mysql", + "description": "Daily active users by team over the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "mysql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.day as time, ut.team_slug as metric, COUNT(DISTINCT ut.user_id) as value FROM _tool_copilot_user_teams ut WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.day, ut.team_slug ORDER BY ut.day", + "refId": "A" + } + ], + "title": "Daily Active Team Users", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "copilot", + "team", + "gh-copilot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "mysql", + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "mysql", + "definition": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Copilot Adoption by Team", + "uid": "github-copilot-adoption-by-team", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/mysql/github-copilot-adoption.json b/grafana/dashboards/mysql/github-copilot-adoption.json new file mode 100644 index 00000000000..e018b6ede64 --- /dev/null +++ b/grafana/dashboards/mysql/github-copilot-adoption.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"links":[],"liveNow":false,"panels":[{"datasource":"mysql","description":"Latest daily active users count","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":50},{"color":"red","value":100}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":0},"id":1,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT daily_active_users as \"Daily Active Users\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY day DESC LIMIT 1","refId":"A"}],"title":"Daily Active Users","type":"stat"},{"datasource":"mysql","description":"Code acceptance rate across the selected time range","fieldConfig":{"defaults":{"mappings":[],"min":0,"max":100,"unit":"percent","thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":20},{"color":"green","value":40}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":0},"id":2,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"Code Acceptance Rate","type":"gauge"},{"datasource":"mysql","description":"Total lines of code added by Copilot in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":12,"y":0},"id":3,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT SUM(loc_added_sum) as \"Lines Added\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"Lines of Code Added","type":"stat"},{"datasource":"mysql","description":"Total pull requests created by Copilot in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":0},"id":4,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT SUM(pr_total_created_by_copilot) as \"PRs by Copilot\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"PRs Created by Copilot","type":"stat"},{"datasource":"mysql","description":"Latest weekly active users count","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":4},"id":15,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(DISTINCT user_login) as \"Weekly Active Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE day >= DATE_SUB(CURDATE(), INTERVAL 7 DAY)\n AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\n","refId":"A"}],"title":"Weekly Active Users (WAU)","type":"stat"},{"datasource":"mysql","description":"Latest monthly active users count","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":4},"id":16,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(DISTINCT user_login) as \"Monthly Active Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE day >= DATE_SUB(CURDATE(), INTERVAL 28 DAY)\n AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\n","refId":"A"}],"title":"Monthly Active Users (MAU)","type":"stat"},{"datasource":"mysql","description":"Total pull requests reviewed by Copilot in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":12,"y":4},"id":17,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT SUM(pr_total_reviewed_by_copilot) as \"PRs Reviewed by Copilot\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"PRs Reviewed by Copilot","type":"stat"},{"datasource":"mysql","description":"Total user-initiated interactions in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"purple","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":4},"id":18,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT SUM(user_initiated_interaction_count) as \"User-Initiated Interactions\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"User-Initiated Interactions","type":"stat"},{"datasource":"mysql","description":"Daily and monthly active users over time","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Users","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"id":5,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time, daily_active_users as \"Daily Active\", monthly_active_users as \"Monthly Active\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"Active Users Over Time","type":"timeseries"},{"datasource":"mysql","description":"Code suggestions generated vs accepted over time","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Count","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"id":6,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time, code_generation_activity_count as \"Suggestions\", code_acceptance_activity_count as \"Acceptances\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"Code Suggestions & Acceptances","type":"timeseries"},{"datasource":"mysql","description":"Lines of code suggested to add versus lines added over time","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Lines of Code","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"id":19,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time, loc_suggested_to_add_sum as \"LOC Suggested\", loc_added_sum as \"LOC Added\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"LOC Suggested vs LOC Added Over Time","type":"timeseries"},{"datasource":"mysql","description":"Pull request activity over time","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Pull Requests","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"id":20,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time, pr_total_created as \"PRs Created\", pr_total_created_by_copilot as \"PRs Created by Copilot\", pr_total_reviewed as \"PRs Reviewed\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"PR Activity Over Time","type":"timeseries"},{"datasource":"mysql","description":"Weekly user-initiated interactions by top Copilot features","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Interactions","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":80,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":0,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"mode":"normal","group":"A"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"},"stacking":{"mode":"normal","group":"A"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT DATE(DATE_SUB(day, INTERVAL WEEKDAY(day) DAY)) as time,\n CASE\n WHEN feature IN (\n SELECT feature FROM (\n SELECT feature\n FROM _tool_copilot_metrics_by_feature\n WHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY feature\n HAVING SUM(user_initiated_interaction_count) > 0\n ORDER BY SUM(user_initiated_interaction_count) DESC\n LIMIT 4\n ) top_features\n ) THEN REPLACE(REPLACE(feature, 'chat_panel_', ''), '_mode', '')\n ELSE 'Other'\n END as metric,\n SUM(user_initiated_interaction_count) as value\nFROM _tool_copilot_metrics_by_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY time, metric\nORDER BY time","refId":"A"}],"title":"Feature Mix Over Time","type":"timeseries","transformations":[{"id":"prepareTimeSeries","options":{"format":"many"}},{"id":"renameByRegex","options":{"regex":"^value (.+)$","renamePattern":"$1"}}]},{"datasource":"mysql","description":"Weekly user-initiated interactions by top IDEs (Top 5 + Other)","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Weekly Interactions","axisPlacement":"auto","barAlignment":0,"drawStyle":"bars","fillOpacity":80,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":0,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"mode":"normal","group":"A"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"id":8,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"},"stacking":{"mode":"normal","group":"A"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT DATE(DATE_SUB(day, INTERVAL WEEKDAY(day) DAY)) as time,\n CASE\n WHEN ide IN (\n SELECT ide FROM (\n SELECT ide\n FROM _tool_copilot_metrics_by_ide\n WHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY ide\n HAVING SUM(user_initiated_interaction_count) > 0\n ORDER BY SUM(user_initiated_interaction_count) DESC\n LIMIT 4\n ) top_ides\n ) THEN ide\n ELSE 'Other'\n END as metric,\n SUM(user_initiated_interaction_count) as value\nFROM _tool_copilot_metrics_by_ide\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY time, metric\nORDER BY time","refId":"A"}],"title":"IDE Adoption Over Time","type":"timeseries","transformations":[{"id":"prepareTimeSeries","options":{"format":"many"}},{"id":"renameByRegex","options":{"regex":"^value (.+)$","renamePattern":"$1"}}]},{"datasource":"mysql","description":"Top languages ranked by code acceptance rate","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"id":9,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT language as \"Language\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_metrics_by_language_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY language\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) DESC, SUM(code_generation_activity_count) DESC\nLIMIT 15","refId":"A"}],"title":"Top Languages by Acceptance Rate","type":"table"},{"datasource":"mysql","description":"Top models ranked by code acceptance rate","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"id":10,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT model as \"Model\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_metrics_by_model_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY model\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) DESC, SUM(code_generation_activity_count) DESC\nLIMIT 10","refId":"A"}],"title":"Top Models by Acceptance Rate","type":"table"},{"datasource":"mysql","description":"Matrix-style breakdown of suggestions and acceptances by language and feature","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":24,"x":0,"y":48},"id":21,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT language as \"Language\",\n feature as \"Feature\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\"\nFROM _tool_copilot_metrics_by_language_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY language, feature\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY language, SUM(code_generation_activity_count) DESC\nLIMIT 100","refId":"A"}],"title":"Feature x Language Matrix","type":"table"},{"datasource":"mysql","description":"Count of distinct users active in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":56},"id":11,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(DISTINCT user_login) as \"Unique Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"Unique Users (Period)","type":"stat"},{"datasource":"mysql","description":"Users who have used Copilot agent mode","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"purple","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":56},"id":12,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(DISTINCT user_login) as \"Agent Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}' AND used_agent = 1","refId":"A"}],"title":"Agent Mode Adopters","type":"stat"},{"datasource":"mysql","description":"Users who have used Copilot chat","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"orange","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":12,"y":56},"id":13,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(DISTINCT user_login) as \"Chat Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}' AND used_chat = 1","refId":"A"}],"title":"Chat Adopters","type":"stat"},{"datasource":"mysql","description":"Percentage of seats with recorded activity","fieldConfig":{"defaults":{"mappings":[],"min":0,"max":100,"unit":"percent","thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":50},{"color":"green","value":75}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":56},"id":14,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT\n COUNT(CASE WHEN last_activity_at IS NOT NULL THEN 1 END) * 100.0 / NULLIF(COUNT(*), 0) as \"Utilization %\"\nFROM _tool_copilot_seats\nWHERE connection_id = ${connection_id}","refId":"A"}],"title":"Seat Utilization","type":"gauge"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":60},"id":22,"panels":[],"title":"Quality & Efficiency Ratios","type":"row"},{"datasource":"mysql","description":"Daily acceptance rate trend computed as acceptances divided by suggestions","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"mappings":[],"max":1,"min":0,"unit":"percentunit","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":61},"id":23,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time,\n code_acceptance_activity_count / NULLIF(code_generation_activity_count, 0) as \"Acceptance Rate\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"Overall Acceptance Rate Trend","type":"timeseries"},{"datasource":"mysql","description":"Daily LOC yield trend computed as added LOC divided by suggested LOC","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"mappings":[],"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":61},"id":24,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time,\n ROUND(loc_added_sum / NULLIF(loc_suggested_to_add_sum, 0), 2) as \"LOC Yield\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nORDER BY 1","refId":"A"}],"title":"LOC Yield Trend","type":"timeseries"},{"datasource":"mysql","description":"Acceptance ratio by Copilot feature","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":69},"id":25,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT feature as \"Feature\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_metrics_by_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY feature\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) DESC, SUM(code_generation_activity_count) DESC\nLIMIT 20","refId":"A"}],"title":"Acceptance Rate by Feature","type":"table"},{"datasource":"mysql","description":"Acceptance ratio by programming language","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":69},"id":26,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT language as \"Language\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_metrics_by_language_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY language\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) DESC, SUM(code_generation_activity_count) DESC\nLIMIT 20","refId":"A"}],"title":"Acceptance Rate by Language","type":"table"},{"datasource":"mysql","description":"Acceptance ratio by model","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":69},"id":27,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT model as \"Model\",\n SUM(code_generation_activity_count) as \"Suggestions\",\n SUM(code_acceptance_activity_count) as \"Acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"Acceptance Rate %\"\nFROM _tool_copilot_metrics_by_model_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY model\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) DESC, SUM(code_generation_activity_count) DESC\nLIMIT 20","refId":"A"}],"title":"Acceptance Rate by Model","type":"table"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":77},"id":28,"panels":[],"title":"User Behavior","type":"row"},{"datasource":"mysql","description":"Top users ranked by code generation activity in the selected time range","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":78},"id":29,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT user_login as \"user_login\",\n SUM(code_generation_activity_count) as \"suggestions\",\n SUM(code_acceptance_activity_count) as \"acceptances\",\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1) as \"acceptance %\"\nFROM _tool_copilot_user_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY user_login\nHAVING SUM(code_generation_activity_count) > 0\nORDER BY SUM(code_generation_activity_count) DESC\nLIMIT 25","refId":"A"}],"title":"Top Users by Code Generations","type":"table"},{"datasource":"mysql","description":"Daily trend of distinct users engaging with agent mode versus chat","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Users","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":78},"id":30,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT day as time,\n COUNT(DISTINCT CASE WHEN used_agent = 1 THEN user_login END) as \"Agent Users\",\n COUNT(DISTINCT CASE WHEN used_chat = 1 THEN user_login END) as \"Chat Users\"\nFROM _tool_copilot_user_daily_metrics\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'\nGROUP BY day\nORDER BY 1","refId":"A"}],"title":"Agent Users vs Chat Users Trend","type":"timeseries"},{"datasource":"mysql","description":"First-seen daily trend of users becoming active for the first time","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Users","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":24,"x":0,"y":86},"id":31,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT first_day as time,\n COUNT(*) as \"New Active Users\"\nFROM (\n SELECT user_login,\n MIN(day) as first_day\n FROM _tool_copilot_user_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY user_login\n) first_seen\nWHERE $__timeFilter(first_day)\nGROUP BY first_day\nORDER BY 1","refId":"A"}],"title":"New Active Users per Day","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":94},"id":32,"panels":[],"title":"Seat Effectiveness","type":"row"},{"datasource":"mysql","description":"Distribution of seat activity by last recorded editor","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":95},"id":33,"options":{"displayLabels":["name","percent"],"legend":{"displayMode":"list","placement":"right","showLegend":true},"pieType":"pie","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COALESCE(NULLIF(last_activity_editor, ''), 'unknown') as \"Editor\",\n COUNT(*) as \"Seats\"\nFROM _tool_copilot_seats\nWHERE connection_id = ${connection_id} AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')\nGROUP BY COALESCE(NULLIF(last_activity_editor, ''), 'unknown')\nORDER BY COUNT(*) DESC","refId":"A"}],"title":"Seat Activity by Editor","type":"piechart"},{"datasource":"mysql","description":"Total, active (last 30 days), and inactive seats for the selected scope","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":95},"id":34,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COUNT(*) as \"total_seats\",\n SUM(CASE WHEN last_activity_at IS NOT NULL AND last_activity_at >= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 30 DAY) THEN 1 ELSE 0 END) as \"active_seats\",\n SUM(CASE WHEN last_activity_at IS NULL OR last_activity_at < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 30 DAY) THEN 1 ELSE 0 END) as \"inactive_seats\"\nFROM _tool_copilot_seats\nWHERE connection_id = ${connection_id} AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')","refId":"A"}],"title":"Seats Summary","type":"table"},{"datasource":"mysql","description":"Seats with no activity or activity older than 30 days","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":95},"id":35,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT user_login as \"user_login\",\n last_activity_at as \"last_activity_at\",\n plan_type as \"plan_type\"\nFROM _tool_copilot_seats\nWHERE connection_id = ${connection_id}\n AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')\n AND (last_activity_at IS NULL OR last_activity_at < DATE_SUB(UTC_TIMESTAMP(), INTERVAL 30 DAY))\nORDER BY last_activity_at IS NULL DESC, last_activity_at ASC\nLIMIT 100","refId":"A"}],"title":"Inactive Seats","type":"table"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":103},"id":36,"panels":[],"title":"Diagnostics","type":"row"},{"datasource":"mysql","description":"Latest available day in enterprise-level Copilot metrics","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":104},"id":37,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT COALESCE(TIMESTAMPDIFF(DAY, MAX(day), UTC_DATE()), 9999) as \"Days Since Latest Data\"\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"Data Freshness","type":"stat"},{"datasource":"mysql","description":"Share of generated suggestions tied to unknown model/feature taxonomy","fieldConfig":{"defaults":{"mappings":[],"min":0,"max":100,"unit":"percent","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":5},{"color":"red","value":20}]}},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":104},"id":38,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT ROUND(\n SUM(CASE WHEN LOWER(COALESCE(model, '')) = 'unknown' OR LOWER(COALESCE(feature, '')) LIKE '%unknown%' THEN code_generation_activity_count ELSE 0 END) * 100.0\n / NULLIF(SUM(code_generation_activity_count), 0),\n 2\n) as \"Unknown Taxonomy %\"\nFROM _tool_copilot_metrics_by_model_feature\nWHERE $__timeFilter(day) AND connection_id = ${connection_id} AND scope_id = '${scope_id}'","refId":"A"}],"title":"Unknown Taxonomy Share","type":"gauge"},{"datasource":"mysql","description":"Row counts for core Copilot metrics tables","fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"id":39,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT *\nFROM (\n SELECT '_tool_copilot_enterprise_daily_metrics' as table_name,\n COUNT(*) as row_count\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_copilot_user_daily_metrics' as table_name,\n COUNT(*) as row_count\n FROM _tool_copilot_user_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_copilot_metrics_by_feature' as table_name,\n COUNT(*) as row_count\n FROM _tool_copilot_metrics_by_feature\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_copilot_metrics_by_model_feature' as table_name,\n COUNT(*) as row_count\n FROM _tool_copilot_metrics_by_model_feature\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n UNION ALL\n SELECT '_tool_copilot_seats' as table_name,\n COUNT(*) as row_count\n FROM _tool_copilot_seats\n WHERE connection_id = ${connection_id} AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')\n) data_volume\nORDER BY row_count DESC","refId":"A"}],"title":"Data Volume by Table","type":"table"}],"refresh":"","schemaVersion":38,"tags":["copilot","devlake"],"templating":{"list":[{"current":{"selected":false,"text":"","value":""},"datasource":"mysql","definition":"SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1","hide":0,"includeAll":false,"label":"Connection ID","multi":false,"name":"connection_id","options":[],"query":"SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"type":"query"},{"current":{"selected":false,"text":"","value":""},"datasource":"mysql","definition":"SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1","hide":0,"includeAll":false,"label":"Scope ID","multi":false,"name":"scope_id","options":[],"query":"SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"type":"query"}]},"time":{"from":"now-90d","to":"now"},"timepicker":{},"timezone":"utc","title":"GitHub Copilot Adoption","uid":"copilot_adoption","version":1,"weekStart":""} \ No newline at end of file diff --git a/grafana/dashboards/mysql/github-copilot-dora-correlation.json b/grafana/dashboards/mysql/github-copilot-dora-correlation.json new file mode 100644 index 00000000000..72cf5b01c7f --- /dev/null +++ b/grafana/dashboards/mysql/github-copilot-dora-correlation.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"},{"datasource":"mysql","enable":true,"hide":false,"iconColor":"purple","name":"Copilot Implementation Date","rawQuery":"SELECT\n UNIX_TIMESTAMP(implementation_date) * 1000 AS time,\n 'Copilot Rollout' AS text,\n 'Implementation Date' AS title\nFROM _tool_copilot_scopes\nWHERE id = '${scope_id}'\n AND implementation_date IS NOT NULL","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":5,"w":24,"x":0,"y":0},"id":1,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"## GitHub Copilot + DORA Correlation Dashboard\n\n**Purpose**: Analyze the **correlation** between GitHub Copilot adoption levels and engineering productivity metrics.\n\n**How it works**:\n- \ud83d\udcc8 **Primary Analysis**: Continuous correlation showing how DORA metrics trend alongside GitHub Copilot adoption intensity\n- \ud83d\udcca **Adoption Tiers**: Metrics grouped by adoption level (<25%, 25-50%, 50-75%, >75% of seats active)\n- \ud83d\udccd **Optional Milestone**: Configure a rollout date to add annotation markers on charts\n\n**Prerequisites**:\n- Ensure **GitHub Copilot metrics** are being collected via the GitHub Copilot plugin\n- Ensure **PR data** exists in DevLake from GitHub/GitLab plugins\n- For DORA metrics (CFR, MTTR): Configure deployments and incidents\n- For Code Quality: Configure SonarQube integration (optional)\n\n**\u26a0\ufe0f Disclaimer**: Correlation does not imply causation. Other factors may influence the metrics shown.","mode":"markdown"},"pluginVersion":"11.0.0","targets":[{"datasource":{"type":"datasource","uid":"grafana"},"queryType":"randomWalk","refId":"A"}],"title":"Dashboard Introduction","type":"text"},{"datasource":"mysql","description":"Weekly GitHub Copilot adoption percentage (active users / total seats) over time. Higher adoption correlates with expected productivity improvements.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Adoption %","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":20,"gradientMode":"scheme","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null},{"color":"green","value":50}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":14},"id":2,"options":{"legend":{"calcs":["mean","lastNotNull"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, active_users, total_seats,\n ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n ROUND(AVG(adoption_pct), 1) AS adoption_pct,\n ROUND(AVG(active_users), 0) AS avg_active_users,\n ROUND(AVG(total_seats), 0) AS avg_total_seats\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n)\nSELECT\n week_start AS time,\n adoption_pct AS 'Adoption %'\nFROM _adoption_weekly\nORDER BY week_start","refId":"A"}],"title":"GitHub Copilot Adoption Trend","type":"timeseries"},{"datasource":"mysql","description":"Pearson correlation coefficient (r) between GitHub Copilot adoption % and PR cycle time. Negative values indicate inverse correlation (higher adoption \u2192 faster PRs). |r| > 0.7 = strong, 0.3-0.7 = moderate, < 0.3 = weak.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"from":-1,"result":{"color":"green","index":0,"text":"Strong \u2193"},"to":-0.7},"type":"range"},{"options":{"from":-0.7,"result":{"color":"yellow","index":1,"text":"Moderate \u2193"},"to":-0.3},"type":"range"},{"options":{"from":-0.3,"result":{"color":"text","index":2,"text":"Weak"},"to":0.3},"type":"range"},{"options":{"from":0.3,"result":{"color":"orange","index":3,"text":"Moderate \u2191"},"to":0.7},"type":"range"},{"options":{"from":0.7,"result":{"color":"red","index":4,"text":"Strong \u2191"},"to":1},"type":"range"}],"noValue":"\u26a0\ufe0f Insufficient data (need 4+ weeks)","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]},"unit":"none","decimals":2},"overrides":[]},"gridPos":{"h":8,"w":4,"x":20,"y":31},"id":3,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"value_and_name","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_metrics_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_correlation_data AS (\n SELECT\n aw.week_start,\n aw.adoption_pct,\n pm.avg_cycle_time_hours\n FROM _adoption_weekly aw\n INNER JOIN _pr_metrics_weekly pm ON aw.week_start = pm.week_start\n WHERE aw.adoption_pct IS NOT NULL AND pm.avg_cycle_time_hours IS NOT NULL\n),\n_stats AS (\n SELECT\n COUNT(*) AS n,\n AVG(adoption_pct) AS mean_x,\n AVG(avg_cycle_time_hours) AS mean_y,\n STDDEV_POP(adoption_pct) AS stddev_x,\n STDDEV_POP(avg_cycle_time_hours) AS stddev_y\n FROM _correlation_data\n),\n_pearson AS (\n SELECT\n CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.stddev_x = 0 OR s.stddev_y = 0 THEN 0\n ELSE (\n SELECT\n SUM((cd.adoption_pct - s.mean_x) * (cd.avg_cycle_time_hours - s.mean_y)) / (s.n * s.stddev_x * s.stddev_y)\n FROM _correlation_data cd\n )\n END AS r\n FROM _stats s\n)\nSELECT ROUND(r, 2) AS value FROM _pearson","refId":"A"}],"title":"Adoption vs PR Cycle Time (r)","type":"stat"},{"datasource":"mysql","description":"Current week's GitHub Copilot adoption percentage (active users / total seats)","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"No data","thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":25},{"color":"green","value":50},{"color":"dark-green","value":75}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":18,"y":14},"id":5,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"value","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS value\nFROM (\n SELECT daily_active_users AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ORDER BY day DESC LIMIT 1\n) _ent\nUNION ALL\nSELECT ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS value\nFROM (\n SELECT total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ORDER BY date DESC LIMIT 1\n) _org\nLIMIT 1","refId":"A"}],"title":"Current Adoption %","type":"stat"},{"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":8,"w":6,"x":12,"y":10},"id":6,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"### Interpreting Correlation (r)\n\n| Value | Meaning |\n|-------|--------|\n| **-1 to -0.7** | \ud83d\udfe2 Strong inverse (higher adoption \u2192 faster PRs) |\n| **-0.7 to -0.3** | \ud83d\udfe1 Moderate inverse |\n| **-0.3 to 0.3** | \u26aa Weak/No correlation |\n| **0.3 to 0.7** | \ud83d\udfe0 Moderate positive |\n| **0.7 to 1** | \ud83d\udd34 Strong positive (unexpected) |\n\n*Note: r < 0 is expected (adoption \u2191 = cycle time \u2193)*","mode":"markdown"},"pluginVersion":"11.0.0","targets":[{"datasource":{"type":"datasource","uid":"grafana"},"queryType":"randomWalk","refId":"A"}],"title":"\ud83d\udcca How to Read Correlation","type":"text"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":30},"id":100,"panels":[],"title":"PR Velocity Impact","type":"row"},{"datasource":"mysql","description":"Percentage difference in PR cycle time between high adoption weeks (>50%) and low adoption weeks (<50%). Negative values indicate improvement (faster PRs during high adoption).","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"noValue":"\u26a0\ufe0f Insufficient data","thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":0},{"color":"red","value":10}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":4,"x":16,"y":31},"id":4,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"value","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_metrics_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_adoption_pr_joined AS (\n SELECT\n aw.week_start,\n aw.adoption_pct,\n pm.avg_cycle_time_hours,\n CASE\n WHEN aw.adoption_pct >= 50 THEN 'high'\n ELSE 'low'\n END AS adoption_tier\n FROM _adoption_weekly aw\n INNER JOIN _pr_metrics_weekly pm ON aw.week_start = pm.week_start\n),\n_tier_averages AS (\n SELECT\n adoption_tier,\n AVG(avg_cycle_time_hours) AS avg_cycle_time\n FROM _adoption_pr_joined\n GROUP BY adoption_tier\n)\nSELECT\n ROUND(\n ((SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'high') -\n (SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'low')) /\n NULLIF((SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'low'), 0) * 100,\n 1\n ) AS value","refId":"A"}],"title":"PR Cycle Time Change (High vs Low Adoption)","type":"stat"},{"datasource":"mysql","description":"Dual-axis chart showing weekly GitHub Copilot adoption % (left axis) and PR cycle time in hours (right axis). Look for inverse correlation - adoption up, cycle time down.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Adoption %"},"properties":[{"id":"custom.axisPlacement","value":"left"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"max","value":100},{"id":"min","value":0}]},{"matcher":{"id":"byName","options":"PR Cycle Time (hrs)"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"h"},{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":31},"id":10,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n)\nSELECT\n week_start AS time,\n ROUND(adoption_pct, 1) AS 'Adoption %'\nFROM _adoption_weekly\nORDER BY week_start","refId":"Adoption"},{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _pr_metrics_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n)\nSELECT\n week_start AS time,\n ROUND(avg_cycle_time_hours, 1) AS 'PR Cycle Time (hrs)'\nFROM _pr_metrics_weekly\nORDER BY week_start","refId":"PRTime"}],"title":"Adoption vs PR Cycle Time","type":"timeseries"},{"datasource":"mysql","description":"Average PR cycle time by adoption tier. Lower times in higher adoption tiers suggest positive Copilot impact.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":24},{"color":"red","value":48}]},"unit":"h"},"overrides":[]},"gridPos":{"h":8,"w":4,"x":12,"y":31},"id":14,"options":{"displayMode":"gradient","maxVizHeight":300,"minVizHeight":16,"minVizWidth":8,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"transformations":[{"id":"rowsToFields","options":{"labelField":"metric","valueField":"value"}}],"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_metrics_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_adoption_pr_joined AS (\n SELECT\n aw.adoption_pct,\n pm.avg_cycle_time_hours,\n CASE\n WHEN aw.adoption_pct < 25 THEN '1. <25%'\n WHEN aw.adoption_pct < 50 THEN '2. 25-50%'\n WHEN aw.adoption_pct < 75 THEN '3. 50-75%'\n ELSE '4. >75%'\n END AS adoption_tier\n FROM _adoption_weekly aw\n INNER JOIN _pr_metrics_weekly pm ON aw.week_start = pm.week_start\n)\nSELECT\n adoption_tier AS metric,\n ROUND(AVG(avg_cycle_time_hours), 1) AS value\nFROM _adoption_pr_joined\nGROUP BY adoption_tier\nORDER BY adoption_tier","refId":"A"}],"title":"PR Cycle Time by Adoption Tier","type":"bargauge"},{"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":6,"w":4,"x":12,"y":35},"id":15,"options":{"code":{"language":"plaintext","showLineNumbers":false,"showMiniMap":false},"content":"**Cycle Time Components** break down PR lifecycle into three phases:\n\n- **Coding Time**: PR creation \u2192 first commit (development speed)\n- **Pickup Time**: PR creation \u2192 first review (reviewer responsiveness)\n- **Review Time**: First review \u2192 merge (actual review duration)\n\nComparing <50% vs \u226550% adoption periods shows which phase improved most with Copilot. For example, if Coding Time shrinks significantly, Copilot accelerates development.","mode":"markdown"},"pluginVersion":"11.0.0","targets":[{"datasource":{"type":"datasource","uid":"grafana"},"queryType":"randomWalk","refId":"A"}],"title":"Understanding Cycle Time Breakdown","type":"text"},{"datasource":"mysql","description":"Scatter plot showing weekly data points. Each point = one week. X-axis = adoption %, Y-axis = PR cycle time. Downward trend line indicates positive Copilot impact.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"Adoption %","axisPlacement":"auto","hideFrom":{"legend":false,"tooltip":false,"viz":false},"pointSize":{"fixed":10},"scaleDistribution":{"type":"linear"},"show":"points"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":6,"w":12,"x":0,"y":35},"id":11,"options":{"dims":{"x":"adoption_pct"},"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"series":[],"seriesMapping":"auto","tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_metrics_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n)\nSELECT\n ROUND(aw.adoption_pct, 1) AS adoption_pct,\n ROUND(pm.avg_cycle_time_hours, 1) AS cycle_time_hours\nFROM _adoption_weekly aw\nINNER JOIN _pr_metrics_weekly pm ON aw.week_start = pm.week_start\nORDER BY aw.adoption_pct","refId":"A"}],"title":"Adoption vs PR Cycle Time (Scatter)","type":"xychart"},{"datasource":"mysql","description":"PR cycle time breakdown during low adoption periods (<50%). Compare with high adoption to see which components improved.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"mappings":[],"unit":"h"},"overrides":[]},"gridPos":{"h":6,"w":4,"x":16,"y":35},"id":12,"options":{"displayLabels":["name","percent"],"legend":{"displayMode":"table","placement":"right","showLegend":true,"values":["value"]},"pieType":"pie","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_low_adoption_weeks AS (\n SELECT week_start FROM _adoption_weekly WHERE adoption_pct < 50\n)\nSELECT\n 'Coding Time' AS component,\n ROUND(AVG(pr_coding_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _low_adoption_weeks)\nUNION ALL\nSELECT\n 'Pickup Time' AS component,\n ROUND(AVG(pr_pickup_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _low_adoption_weeks)\nUNION ALL\nSELECT\n 'Review Time' AS component,\n ROUND(AVG(pr_review_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _low_adoption_weeks)","refId":"A"}],"title":"Cycle Time Components: Low Adoption","type":"piechart"},{"datasource":"mysql","description":"PR cycle time breakdown during high adoption periods (>=50%). Compare with low adoption to see which components improved.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"mappings":[],"unit":"h"},"overrides":[]},"gridPos":{"h":6,"w":4,"x":20,"y":35},"id":13,"options":{"displayLabels":["name","percent"],"legend":{"displayMode":"table","placement":"right","showLegend":true,"values":["value"]},"pieType":"pie","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_high_adoption_weeks AS (\n SELECT week_start FROM _adoption_weekly WHERE adoption_pct >= 50\n)\nSELECT\n 'Coding Time' AS component,\n ROUND(AVG(pr_coding_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _high_adoption_weeks)\nUNION ALL\nSELECT\n 'Pickup Time' AS component,\n ROUND(AVG(pr_pickup_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _high_adoption_weeks)\nUNION ALL\nSELECT\n 'Review Time' AS component,\n ROUND(AVG(pr_review_time) / 60.0, 1) AS value\nFROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) IN (SELECT week_start FROM _high_adoption_weeks)","refId":"A"}],"title":"Cycle Time Components: High Adoption","type":"piechart"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":45},"id":200,"panels":[],"title":"Deployment Frequency Impact","type":"row"},{"datasource":"mysql","description":"Dual-axis chart showing GitHub Copilot adoption percentage and deployment frequency over time. Look for correlation patterns - does higher adoption correlate with more frequent deployments?","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Adoption %"},"properties":[{"id":"custom.axisPlacement","value":"left"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"min","value":0},{"id":"max","value":100}]},{"matcher":{"id":"byName","options":"Deployments"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"short"},{"id":"color","value":{"fixedColor":"orange","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":46},"id":31,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS deploy_count\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT\n UNIX_TIMESTAMP(aw.week_start) AS time_sec,\n aw.adoption_pct AS 'Adoption %',\n COALESCE(dw.deploy_count, 0) AS 'Deployments'\nFROM _adoption_weekly aw\nLEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\nORDER BY aw.week_start","refId":"A"}],"title":"Adoption vs Deployment Frequency","type":"timeseries"},{"datasource":"mysql","description":"Deployment frequency grouped by GitHub Copilot adoption tier. Higher adoption should correlate with more frequent deployments if Copilot accelerates development velocity.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"yellow","value":null},{"color":"green","value":5}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":46},"id":32,"options":{"displayMode":"gradient","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["sum"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"transformations":[{"id":"rowsToFields","options":{"labelField":"Adoption Tier","valueField":"Deploys/Week"}}],"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct,\n CASE\n WHEN AVG(adoption_pct) < 25 THEN '1: <25%'\n WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%'\n WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%'\n ELSE '4: >75%'\n END AS adoption_tier\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS deploy_count\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT\n aw.adoption_tier AS 'Adoption Tier',\n ROUND(AVG(COALESCE(dw.deploy_count, 0)), 1) AS 'Deploys/Week'\nFROM _adoption_weekly aw\nLEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\nGROUP BY aw.adoption_tier\nORDER BY aw.adoption_tier","refId":"A"}],"title":"Deployments per Week by Adoption Tier","type":"bargauge"},{"datasource":"mysql","description":"Pearson correlation coefficient (r) between GitHub Copilot adoption % and weekly deployment count. Positive values indicate higher adoption correlates with more deployments.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":0.3},{"color":"green","value":0.7}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":46},"id":33,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS deploy_count\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n),\n_joined AS (\n SELECT\n aw.adoption_pct,\n COALESCE(dw.deploy_count, 0) AS deploy_count\n FROM _adoption_weekly aw\n LEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\n)\nSELECT\n ROUND(\n (COUNT(*) * SUM(adoption_pct * deploy_count) - SUM(adoption_pct) * SUM(deploy_count)) /\n NULLIF(\n SQRT(\n (COUNT(*) * SUM(adoption_pct * adoption_pct) - POW(SUM(adoption_pct), 2)) *\n (COUNT(*) * SUM(deploy_count * deploy_count) - POW(SUM(deploy_count), 2))\n ),\n 0\n ),\n 2\n ) AS correlation_r\nFROM _joined\nWHERE adoption_pct IS NOT NULL","refId":"A"}],"title":"Adoption vs Deployment Frequency (r)","type":"stat"},{"datasource":"mysql","description":"Percentage change in deployment frequency between low adoption (<50%) and high adoption (\u226550%) periods.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":0},{"color":"green","value":20}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":50},"id":34,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS deploy_count\n FROM cicd_deployment_commits\n WHERE result = 'SUCCESS'\n AND $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n),\n_adoption_deploy AS (\n SELECT\n CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier,\n COALESCE(dw.deploy_count, 0) AS deploy_count\n FROM _adoption_weekly aw\n LEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\n),\n_tier_avg AS (\n SELECT\n tier,\n AVG(deploy_count) AS avg_deploys\n FROM _adoption_deploy\n GROUP BY tier\n)\nSELECT\n ROUND(\n ((SELECT avg_deploys FROM _tier_avg WHERE tier = 'high') -\n (SELECT avg_deploys FROM _tier_avg WHERE tier = 'low')) /\n NULLIF((SELECT avg_deploys FROM _tier_avg WHERE tier = 'low'), 0) * 100,\n 1\n ) AS change_pct","refId":"A"}],"title":"Deployment Frequency Change (High vs Low Adoption)","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":5},"id":250,"panels":[],"title":"Adoption Intensity Analysis","type":"row"},{"datasource":"mysql","description":"Comprehensive comparison of all DORA metrics across GitHub Copilot adoption tiers. Lower values (green) are better for cycle time, CFR, and MTTR. Higher values (green) are better for deployment frequency.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","cellOptions":{"type":"auto"},"inspect":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":24,"x":0,"y":6},"id":72,"options":{"cellHeight":"sm","footer":{"countRows":false,"fields":"","reducer":["sum"],"show":false},"showHeader":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_adoption_tiers AS (\n SELECT\n week_start,\n adoption_pct,\n CASE\n WHEN adoption_pct < 25 THEN '<25%'\n WHEN adoption_pct < 50 THEN '25-50%'\n WHEN adoption_pct < 75 THEN '50-75%'\n ELSE '>75%'\n END AS tier\n FROM _adoption_weekly\n),\n_pr_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time / 60) AS cycle_time_hrs\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND pr_merged_date IS NOT NULL AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_deploy_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS deploy_count,\n SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys\n FROM cicd_deployment_commits\n WHERE $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n),\n_cfr_weekly AS (\n SELECT\n week_start,\n ROUND(failed_deploys * 100.0 / NULLIF(deploy_count, 0), 1) AS cfr_pct\n FROM _deploy_weekly\n),\n_mttr_weekly AS (\n SELECT\n DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n AVG(TIMESTAMPDIFF(HOUR, created_date, resolution_date)) AS mttr_hours\n FROM issues\n WHERE type = 'INCIDENT'\n AND resolution_date IS NOT NULL\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n),\n_tier_metrics AS (\n SELECT\n at.tier,\n ROUND(AVG(pw.cycle_time_hrs), 1) AS pr_cycle_time,\n ROUND(AVG(dw.deploy_count), 1) AS deploy_freq,\n ROUND(AVG(cfr.cfr_pct), 1) AS cfr_pct,\n ROUND(AVG(mttr.mttr_hours), 1) AS mttr_hours\n FROM _adoption_tiers at\n LEFT JOIN _pr_weekly pw ON at.week_start = pw.week_start\n LEFT JOIN _deploy_weekly dw ON at.week_start = dw.week_start\n LEFT JOIN _cfr_weekly cfr ON at.week_start = cfr.week_start\n LEFT JOIN _mttr_weekly mttr ON at.week_start = mttr.week_start\n GROUP BY at.tier\n)\nSELECT\n tier AS 'Adoption Tier',\n COALESCE(pr_cycle_time, 0) AS 'PR Cycle Time (hrs)',\n COALESCE(deploy_freq, 0) AS 'Deploys/Week',\n COALESCE(cfr_pct, 0) AS 'CFR %',\n COALESCE(mttr_hours, 0) AS 'MTTR (hours)'\nFROM _tier_metrics\nORDER BY FIELD(tier, '<25%', '25-50%', '50-75%', '>75%')","refId":"A"}],"title":"DORA Metrics by Adoption Tier","type":"table"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":54},"id":300,"panels":[],"title":"Change Failure Rate Impact","type":"row"},{"datasource":"mysql","description":"Dual-axis chart showing GitHub Copilot adoption percentage and Change Failure Rate over time. CFR should decrease (improve) as adoption increases if Copilot improves code quality.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Adoption %"},"properties":[{"id":"custom.axisPlacement","value":"left"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"min","value":0},{"id":"max","value":100}]},{"matcher":{"id":"byName","options":"CFR %"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"red","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":55},"id":41,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS total_deploys,\n SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys\n FROM cicd_deployment_commits\n WHERE $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT\n UNIX_TIMESTAMP(aw.week_start) AS time_sec,\n aw.adoption_pct AS 'Adoption %',\n ROUND(dw.failed_deploys * 100.0 / NULLIF(dw.total_deploys, 0), 1) AS 'CFR %'\nFROM _adoption_weekly aw\nLEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\nORDER BY aw.week_start","refId":"A"}],"title":"Adoption vs Change Failure Rate","type":"timeseries"},{"datasource":"mysql","description":"Change Failure Rate grouped by GitHub Copilot adoption tier. Lower CFR at higher adoption indicates Copilot improves code quality and reduces failures.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":10},{"color":"red","value":15}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":55},"id":42,"options":{"displayMode":"gradient","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"transformations":[{"id":"rowsToFields","options":{"labelField":"Adoption Tier","valueField":"CFR %"}}],"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct,\n CASE\n WHEN AVG(adoption_pct) < 25 THEN '1: <25%'\n WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%'\n WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%'\n ELSE '4: >75%'\n END AS adoption_tier\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS total_deploys,\n SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys\n FROM cicd_deployment_commits\n WHERE $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n)\nSELECT\n aw.adoption_tier AS 'Adoption Tier',\n ROUND(AVG(dw.failed_deploys * 100.0 / NULLIF(dw.total_deploys, 0)), 1) AS 'CFR %'\nFROM _adoption_weekly aw\nLEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\nGROUP BY aw.adoption_tier\nORDER BY aw.adoption_tier","refId":"A"}],"title":"CFR by Adoption Tier","type":"bargauge"},{"datasource":"mysql","description":"Pearson correlation coefficient (r) between GitHub Copilot adoption % and Change Failure Rate. NEGATIVE values are good - they indicate higher adoption correlates with lower failure rates.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-0.3},{"color":"red","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":55},"id":43,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS total_deploys,\n SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys\n FROM cicd_deployment_commits\n WHERE $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n),\n_joined AS (\n SELECT\n aw.adoption_pct,\n (dw.failed_deploys * 100.0 / NULLIF(dw.total_deploys, 0)) AS cfr_pct\n FROM _adoption_weekly aw\n LEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\n WHERE dw.total_deploys > 0\n)\nSELECT\n ROUND(\n (COUNT(*) * SUM(adoption_pct * cfr_pct) - SUM(adoption_pct) * SUM(cfr_pct)) /\n NULLIF(\n SQRT(\n (COUNT(*) * SUM(adoption_pct * adoption_pct) - POW(SUM(adoption_pct), 2)) *\n (COUNT(*) * SUM(cfr_pct * cfr_pct) - POW(SUM(cfr_pct), 2))\n ),\n 0\n ),\n 2\n ) AS correlation_r\nFROM _joined","refId":"A"}],"title":"Adoption vs CFR (r)","type":"stat"},{"datasource":"mysql","description":"Percentage change in Change Failure Rate between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate improvement.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-10},{"color":"red","value":0}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":59},"id":44,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_deployments_weekly AS (\n SELECT\n DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY) AS week_start,\n COUNT(*) AS total_deploys,\n SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys\n FROM cicd_deployment_commits\n WHERE $__timeFilter(finished_date)\n GROUP BY DATE_SUB(DATE(finished_date), INTERVAL WEEKDAY(DATE(finished_date)) DAY)\n),\n_adoption_cfr AS (\n SELECT\n CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier,\n (dw.failed_deploys * 100.0 / NULLIF(dw.total_deploys, 0)) AS cfr_pct\n FROM _adoption_weekly aw\n LEFT JOIN _deployments_weekly dw ON aw.week_start = dw.week_start\n WHERE dw.total_deploys > 0\n),\n_tier_avg AS (\n SELECT\n tier,\n AVG(cfr_pct) AS avg_cfr\n FROM _adoption_cfr\n GROUP BY tier\n)\nSELECT\n ROUND(\n ((SELECT avg_cfr FROM _tier_avg WHERE tier = 'high') -\n (SELECT avg_cfr FROM _tier_avg WHERE tier = 'low')) /\n NULLIF((SELECT avg_cfr FROM _tier_avg WHERE tier = 'low'), 0) * 100,\n 1\n ) AS change_pct","refId":"A"}],"title":"CFR Change (High vs Low Adoption)","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":63},"id":400,"panels":[],"title":"Recovery Time (MTTR) Impact","type":"row"},{"datasource":"mysql","description":"Dual-axis chart showing GitHub Copilot adoption percentage and Mean Time to Recovery (MTTR) over time. MTTR should decrease (improve) as adoption increases if Copilot helps faster incident resolution.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Adoption %"},"properties":[{"id":"custom.axisPlacement","value":"left"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"min","value":0},{"id":"max","value":100}]},{"matcher":{"id":"byName","options":"MTTR (hours)"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"h"},{"id":"color","value":{"fixedColor":"purple","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"id":51,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n)\nSELECT\n week_start AS time,\n ROUND(adoption_pct, 1) AS 'Adoption %'\nFROM _adoption_weekly\nORDER BY week_start","refId":"Adoption"},{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _incidents_weekly AS (\n SELECT\n DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n AVG(TIMESTAMPDIFF(HOUR, created_date, resolution_date)) AS mttr_hours\n FROM issues\n WHERE type = 'INCIDENT'\n AND resolution_date IS NOT NULL\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT\n week_start AS time,\n ROUND(mttr_hours, 1) AS 'MTTR (hours)'\nFROM _incidents_weekly\nORDER BY week_start","refId":"MTTR"}],"title":"Adoption vs MTTR","type":"timeseries"},{"datasource":"mysql","description":"Mean Time to Recovery grouped by GitHub Copilot adoption tier. Lower MTTR at higher adoption indicates Copilot helps faster incident resolution.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":24},{"color":"red","value":72}]},"unit":"h"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":64},"id":52,"options":{"displayMode":"gradient","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"transformations":[{"id":"rowsToFields","options":{"labelField":"Adoption Tier","valueField":"MTTR (hours)"}}],"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct,\n CASE\n WHEN AVG(adoption_pct) < 25 THEN '1: <25%'\n WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%'\n WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%'\n ELSE '4: >75%'\n END AS adoption_tier\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_incidents_weekly AS (\n SELECT\n DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n AVG(TIMESTAMPDIFF(HOUR, created_date, resolution_date)) AS mttr_hours\n FROM issues\n WHERE type = 'INCIDENT'\n AND resolution_date IS NOT NULL\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n)\nSELECT\n aw.adoption_tier AS 'Adoption Tier',\n ROUND(AVG(iw.mttr_hours), 1) AS 'MTTR (hours)'\nFROM _adoption_weekly aw\nLEFT JOIN _incidents_weekly iw ON aw.week_start = iw.week_start\nWHERE iw.mttr_hours IS NOT NULL\nGROUP BY aw.adoption_tier\nORDER BY aw.adoption_tier","refId":"A"}],"title":"MTTR by Adoption Tier","type":"bargauge"},{"datasource":"mysql","description":"Pearson correlation coefficient (r) between GitHub Copilot adoption % and MTTR. NEGATIVE values are good - they indicate higher adoption correlates with faster recovery.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-0.3},{"color":"red","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":64},"id":53,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_incidents_weekly AS (\n SELECT\n DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n AVG(TIMESTAMPDIFF(HOUR, created_date, resolution_date)) AS mttr_hours\n FROM issues\n WHERE type = 'INCIDENT'\n AND resolution_date IS NOT NULL\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n),\n_joined AS (\n SELECT\n aw.adoption_pct,\n iw.mttr_hours\n FROM _adoption_weekly aw\n INNER JOIN _incidents_weekly iw ON aw.week_start = iw.week_start\n)\nSELECT\n ROUND(\n (COUNT(*) * SUM(adoption_pct * mttr_hours) - SUM(adoption_pct) * SUM(mttr_hours)) /\n NULLIF(\n SQRT(\n (COUNT(*) * SUM(adoption_pct * adoption_pct) - POW(SUM(adoption_pct), 2)) *\n (COUNT(*) * SUM(mttr_hours * mttr_hours) - POW(SUM(mttr_hours), 2))\n ),\n 0\n ),\n 2\n ) AS correlation_r\nFROM _joined","refId":"A"}],"title":"Adoption vs MTTR (r)","type":"stat"},{"datasource":"mysql","description":"Percentage change in MTTR between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate faster recovery.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-20},{"color":"red","value":0}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":68},"id":54,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_incidents_weekly AS (\n SELECT\n DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY) AS week_start,\n AVG(TIMESTAMPDIFF(HOUR, created_date, resolution_date)) AS mttr_hours\n FROM issues\n WHERE type = 'INCIDENT'\n AND resolution_date IS NOT NULL\n AND $__timeFilter(resolution_date)\n GROUP BY DATE_SUB(DATE(resolution_date), INTERVAL WEEKDAY(DATE(resolution_date)) DAY)\n),\n_adoption_mttr AS (\n SELECT\n CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier,\n iw.mttr_hours\n FROM _adoption_weekly aw\n INNER JOIN _incidents_weekly iw ON aw.week_start = iw.week_start\n),\n_tier_avg AS (\n SELECT\n tier,\n AVG(mttr_hours) AS avg_mttr\n FROM _adoption_mttr\n GROUP BY tier\n)\nSELECT\n ROUND(\n ((SELECT avg_mttr FROM _tier_avg WHERE tier = 'high') -\n (SELECT avg_mttr FROM _tier_avg WHERE tier = 'low')) /\n NULLIF((SELECT avg_mttr FROM _tier_avg WHERE tier = 'low'), 0) * 100,\n 1\n ) AS change_pct","refId":"A"}],"title":"MTTR Change (High vs Low Adoption)","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":72},"id":500,"panels":[],"title":"Code Review Time Impact","type":"row"},{"datasource":"mysql","description":"Code review time (time from first review request to merge) grouped by GitHub Copilot adoption tier. Shorter review times at higher adoption may indicate Copilot-assisted code is easier to review.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":24},{"color":"red","value":48}]},"unit":"h"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":12,"y":73},"id":61,"options":{"displayMode":"gradient","maxVizHeight":300,"minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"transformations":[{"id":"rowsToFields","options":{"labelField":"Adoption Tier","valueField":"Review Time (hours)"}}],"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct,\n CASE\n WHEN AVG(adoption_pct) < 25 THEN '1: <25%'\n WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%'\n WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%'\n ELSE '4: >75%'\n END AS adoption_tier\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_review_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_review_time) / 60.0 AS avg_review_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n AND pr_review_time > 0\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n)\nSELECT\n aw.adoption_tier AS 'Adoption Tier',\n ROUND(AVG(prw.avg_review_time_hours), 1) AS 'Review Time (hours)'\nFROM _adoption_weekly aw\nLEFT JOIN _pr_review_weekly prw ON aw.week_start = prw.week_start\nWHERE prw.avg_review_time_hours IS NOT NULL\nGROUP BY aw.adoption_tier\nORDER BY aw.adoption_tier","refId":"A"}],"title":"Review Time by Adoption Tier","type":"bargauge"},{"datasource":"mysql","description":"Dual-axis chart showing GitHub Copilot adoption % and average code review time over time. Look for patterns indicating whether Copilot-assisted PRs get reviewed faster.","fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[{"matcher":{"id":"byName","options":"Adoption %"},"properties":[{"id":"custom.axisPlacement","value":"left"},{"id":"unit","value":"percent"},{"id":"color","value":{"fixedColor":"blue","mode":"fixed"}},{"id":"min","value":0},{"id":"max","value":100}]},{"matcher":{"id":"byName","options":"Review Time (h)"},"properties":[{"id":"custom.axisPlacement","value":"right"},{"id":"unit","value":"h"},{"id":"color","value":{"fixedColor":"green","mode":"fixed"}}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":73},"id":62,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"time_series","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_review_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_review_time) / 60.0 AS avg_review_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n AND pr_review_time > 0\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n)\nSELECT\n UNIX_TIMESTAMP(aw.week_start) AS time_sec,\n aw.adoption_pct AS 'Adoption %',\n ROUND(prw.avg_review_time_hours, 1) AS 'Review Time (h)'\nFROM _adoption_weekly aw\nLEFT JOIN _pr_review_weekly prw ON aw.week_start = prw.week_start\nORDER BY aw.week_start","refId":"A"}],"title":"Adoption vs Review Time Trend","type":"timeseries"},{"datasource":"mysql","description":"Percentage change in code review time between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate faster reviews.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-10},{"color":"red","value":0}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":77},"id":63,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_review_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_review_time) / 60.0 AS avg_review_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n AND pr_review_time > 0\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_adoption_review AS (\n SELECT\n CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier,\n prw.avg_review_time_hours\n FROM _adoption_weekly aw\n INNER JOIN _pr_review_weekly prw ON aw.week_start = prw.week_start\n),\n_tier_avg AS (\n SELECT\n tier,\n AVG(avg_review_time_hours) AS avg_review\n FROM _adoption_review\n GROUP BY tier\n)\nSELECT\n ROUND(\n ((SELECT avg_review FROM _tier_avg WHERE tier = 'high') -\n (SELECT avg_review FROM _tier_avg WHERE tier = 'low')) /\n NULLIF((SELECT avg_review FROM _tier_avg WHERE tier = 'low'), 0) * 100,\n 1\n ) AS change_pct","refId":"A"}],"title":"Review Time Change (High vs Low Adoption)","type":"stat"},{"datasource":"mysql","description":"Pearson correlation coefficient (r) between GitHub Copilot adoption % and code review time. NEGATIVE values indicate higher adoption correlates with faster reviews.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":-0.3},{"color":"red","value":0}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":18,"y":73},"id":64,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(date)\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_pr_review_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_review_time) / 60.0 AS avg_review_time_hours\n FROM project_pr_metrics\n WHERE project_name = ${project:sqlstring}\n AND $__timeFilter(pr_merged_date)\n AND pr_review_time > 0\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT\n aw.adoption_pct,\n prw.avg_review_time_hours\n FROM _adoption_weekly aw\n INNER JOIN _pr_review_weekly prw ON aw.week_start = prw.week_start\n)\nSELECT\n ROUND(\n (COUNT(*) * SUM(adoption_pct * avg_review_time_hours) - SUM(adoption_pct) * SUM(avg_review_time_hours)) /\n NULLIF(\n SQRT(\n (COUNT(*) * SUM(adoption_pct * adoption_pct) - POW(SUM(adoption_pct), 2)) *\n (COUNT(*) * SUM(avg_review_time_hours * avg_review_time_hours) - POW(SUM(avg_review_time_hours), 2))\n ),\n 0\n ),\n 2\n ) AS correlation_r\nFROM _joined","refId":"A"}],"title":"Adoption vs Review Time (r)","type":"stat"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":81},"id":600,"panels":[],"title":"Code Quality Impact (Optional - Requires SonarQube)","type":"row"},{"datasource":"mysql","description":"Bug count per file by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":1},{"color":"red","value":5}]}},"overrides":[]},"gridPos":{"h":6,"w":6,"x":0,"y":82},"id":80,"options":{"displayMode":"gradient","minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"WITH _copilot_adoption AS (\n SELECT metric_date, ROUND(active_users * 100.0 / NULLIF(total_seats, 0), 1) AS adoption_pct\n FROM (\n SELECT day AS metric_date, SUM(daily_active_users) AS active_users,\n (SELECT COUNT(*) FROM _tool_copilot_seats s WHERE s.connection_id = ${connection_id}\n AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes sc\n WHERE sc.connection_id = ${connection_id} AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n GROUP BY day\n UNION ALL\n SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats\n FROM _tool_copilot_org_daily_metrics\n WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}'\n ) _unified),\n_adoption_weekly AS (\n SELECT\n DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY) AS week_start,\n AVG(adoption_pct) AS adoption_pct\n FROM _copilot_adoption\n GROUP BY DATE_SUB(metric_date, INTERVAL WEEKDAY(metric_date) DAY)\n),\n_adoption_tiers AS (\n SELECT\n week_start,\n CASE\n WHEN adoption_pct < 25 THEN '<25%'\n WHEN adoption_pct < 50 THEN '25-50%'\n WHEN adoption_pct < 75 THEN '50-75%'\n ELSE '>75%'\n END AS tier\n FROM _adoption_weekly\n)\nSELECT 'N/A - Configure SonarQube' AS Tier, 0 AS 'Bugs/File'\nFROM (SELECT 1) AS d\nWHERE NOT EXISTS (SELECT 1 FROM cq_file_metrics LIMIT 1)","refId":"A"}],"title":"Bugs per File by Adoption","type":"bargauge"},{"datasource":"mysql","description":"Code smell count per file by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":5},{"color":"red","value":20}]}},"overrides":[]},"gridPos":{"h":6,"w":6,"x":6,"y":82},"id":81,"options":{"displayMode":"gradient","minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT 'N/A - Configure SonarQube' AS Tier, 0 AS 'Code Smells/File'\nFROM (SELECT 1) AS d\nWHERE NOT EXISTS (SELECT 1 FROM cq_file_metrics LIMIT 1)","refId":"A"}],"title":"Code Smells by Adoption","type":"bargauge"},{"datasource":"mysql","description":"Code complexity by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"yellow","value":10},{"color":"red","value":25}]}},"overrides":[]},"gridPos":{"h":6,"w":6,"x":12,"y":82},"id":82,"options":{"displayMode":"gradient","minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT 'N/A - Configure SonarQube' AS Tier, 0 AS Complexity\nFROM (SELECT 1) AS d\nWHERE NOT EXISTS (SELECT 1 FROM cq_file_metrics LIMIT 1)","refId":"A"}],"title":"Complexity by Adoption","type":"bargauge"},{"datasource":"mysql","description":"Code coverage percentage by GitHub Copilot adoption tier. Higher is better. Requires SonarQube integration.","fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"max":100,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null},{"color":"yellow","value":50},{"color":"green","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":6,"w":6,"x":18,"y":82},"id":83,"options":{"displayMode":"gradient","minVizHeight":10,"minVizWidth":0,"namePlacement":"auto","orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"sizing":"auto","valueMode":"color"},"pluginVersion":"11.0.0","targets":[{"datasource":"mysql","format":"table","rawQuery":true,"rawSql":"SELECT 'N/A - Configure SonarQube' AS Tier, 0 AS Coverage\nFROM (SELECT 1) AS d\nWHERE NOT EXISTS (SELECT 1 FROM cq_file_metrics LIMIT 1)","refId":"A"}],"title":"Coverage by Adoption","type":"bargauge"}],"refresh":"","schemaVersion":38,"tags":["copilot","devlake","impact","dora"],"templating":{"list":[{"current":{"selected":false,"text":"","value":""},"datasource":"mysql","definition":"SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY 1","hide":0,"includeAll":false,"label":"Connection ID","multi":false,"name":"connection_id","options":[],"query":"SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"type":"query"},{"current":{"selected":false,"text":"","value":""},"datasource":"mysql","definition":"SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${connection_id}' AS UNSIGNED) ORDER BY 1","hide":0,"includeAll":false,"label":"Scope ID (Organization)","multi":false,"name":"scope_id","options":[],"query":"SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${connection_id}' AS UNSIGNED) ORDER BY 1","refresh":2,"regex":"","skipUrlSync":false,"sort":0,"type":"query"},{"current":{"selected":false,"text":"","value":""},"datasource":"mysql","definition":"SELECT DISTINCT project_name FROM project_pr_metrics ORDER BY 1","hide":0,"includeAll":false,"label":"Project","multi":false,"name":"project","options":[],"query":"SELECT DISTINCT project_name FROM project_pr_metrics ORDER BY 1","refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"selected":false,"text":"2023","value":"2023"},"hide":0,"includeAll":false,"label":"DORA Report","multi":false,"name":"dora_report","options":[{"selected":false,"text":"2021","value":"2021"},{"selected":true,"text":"2023","value":"2023"}],"query":"2021, 2023","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-90d","to":"now"},"timepicker":{},"timezone":"utc","title":"GitHub Copilot + DORA Correlation","uid":"copilot_impact","version":1,"weekStart":""} \ No newline at end of file diff --git a/grafana/dashboards/GithubReleaseQualityAndContributionAnalysis.json b/grafana/dashboards/mysql/github-release-quality-and-contribution-analysis.json similarity index 97% rename from grafana/dashboards/GithubReleaseQualityAndContributionAnalysis.json rename to grafana/dashboards/mysql/github-release-quality-and-contribution-analysis.json index f2c354145b5..74bcc745d23 100644 --- a/grafana/dashboards/GithubReleaseQualityAndContributionAnalysis.json +++ b/grafana/dashboards/mysql/github-release-quality-and-contribution-analysis.json @@ -33,7 +33,7 @@ "content": "- Use Cases: This dashboard can be used to track bugs.\n- Data Source Required: GitHub ([transformation](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional) required. Additional settings are required to get version data).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -58,7 +58,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -146,7 +146,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -273,7 +273,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -359,7 +359,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -515,7 +515,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -601,7 +601,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -727,7 +727,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -763,7 +763,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -850,7 +850,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -937,7 +937,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -989,7 +989,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1025,7 +1025,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1123,7 +1123,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1177,7 +1177,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1213,7 +1213,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1317,7 +1317,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1371,7 +1371,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1407,7 +1407,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1539,7 +1539,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1575,7 +1575,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1688,7 +1688,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1724,7 +1724,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1850,7 +1850,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1900,7 +1900,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1995,7 +1995,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2071,7 +2071,7 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -2106,7 +2106,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2223,7 +2223,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2340,7 +2340,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2457,7 +2457,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2509,7 +2509,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -2545,7 +2545,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2652,7 +2652,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "queryType": "randomWalk", @@ -2681,7 +2681,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from repos", "description": null, "error": null, @@ -2706,7 +2706,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "GitHub_Release_Quality_and_Contribution_Analysis", + "title": "GitHub Release Quality and Contribution Analysis", "uid": "2xuOaQUnk4", "version": 3 -} \ No newline at end of file +} diff --git a/grafana/dashboards/mysql/kiro-credits-dora.json b/grafana/dashboards/mysql/kiro-credits-dora.json new file mode 100644 index 00000000000..46614cc0834 --- /dev/null +++ b/grafana/dashboards/mysql/kiro-credits-dora.json @@ -0,0 +1,426 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "DORA Dashboard", + "type": "link", + "url": "/d/qNo8_0M4z/dora" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "## Kiro Credits + DORA Correlation\nThis dashboard correlates **Kiro AI usage (credits and messages)** with **DORA** performance indicators at a weekly aggregate level.\n\n- **Pearson's r** measures linear correlation: negative r suggests higher AI usage may correlate with shorter cycle times.\n- Data is aggregated by **week** and joined on `week_start`.", + "mode": "markdown" + }, + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "id": 2, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Pearson correlation coefficient between weekly Kiro credits and PR cycle time", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [{ "options": { "match": "null", "result": { "text": "Need more data" } }, "type": "special" }], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": -0.3 }, + { "color": "orange", "value": 0.3 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 4 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (\n SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_pr_weekly AS (\n SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_cycle_hours\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL\n AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y\n FROM _kiro_weekly k\n INNER JOIN _pr_weekly p ON k.week_start = p.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy\n FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", + "refId": "A" + } + ], + "title": "Credits vs Cycle Time (r)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Total Kiro credits consumed in period", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 4, "x": 8, "y": 4 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(credits_used) AS 'Credits Used'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Median PR cycle time in hours", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 4, "x": 12, "y": 4 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _ranked AS (\n SELECT pr_cycle_time / 60.0 AS ct,\n PERCENT_RANK() OVER (ORDER BY pr_cycle_time) AS prank\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\n)\nSELECT ROUND(MAX(ct), 1) AS 'Median Cycle Time'\nFROM _ranked WHERE prank <= 0.5", + "refId": "A" + } + ], + "title": "Median Cycle Time", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Cycle time comparison: weeks with above-median vs below-median AI credits usage", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 4 }, + "id": 6, + "options": { + "barRadius": 0.1, + "barWidth": 0.6, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_pr_weekly AS (\n SELECT DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS week_start,\n AVG(pr_cycle_time) / 60.0 AS avg_ct\n FROM project_pr_metrics\n WHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL AND $__timeFilter(pr_merged_date)\n GROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly k INNER JOIN _pr_weekly p ON k.week_start = p.week_start\n),\n_med AS (\n SELECT weekly_credits AS med FROM _joined ORDER BY weekly_credits LIMIT 1 OFFSET (SELECT FLOOR(COUNT(*)/2) FROM _joined)\n)\nSELECT\n CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS 'Tier',\n ROUND(AVG(j.avg_ct), 1) AS 'Avg Cycle Time'\nFROM _joined j, _med m\nGROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", + "refId": "A" + } + ], + "title": "Cycle Time: High vs Low AI Usage", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly Kiro credits consumed", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, + "id": 11, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS 'Credits Used',\n COUNT(DISTINCT user_id) AS 'Active Users'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Kiro Credits & Active Users", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly average PR cycle time in hours", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, + "id": 12, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY) AS time,\n ROUND(AVG(pr_cycle_time) / 60.0, 1) AS 'Avg Cycle Time (hrs)',\n COUNT(*) AS 'PRs Merged'\nFROM project_pr_metrics\nWHERE pr_merged_date IS NOT NULL AND pr_cycle_time IS NOT NULL\n AND $__timeFilter(pr_merged_date)\nGROUP BY DATE_SUB(DATE(pr_merged_date), INTERVAL WEEKDAY(DATE(pr_merged_date)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly PR Cycle Time & Volume", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, + "id": 20, + "panels": [], + "title": "Deployment Frequency", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Pearson correlation between weekly credits and deployment count", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [{ "options": { "match": "null", "result": { "text": "Need more data" } }, "type": "special" }], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 6, "x": 0, "y": 20 }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS week_start,\n SUM(credits_used) AS weekly_credits\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n),\n_deploy_weekly AS (\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS week_start,\n COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n),\n_joined AS (\n SELECT k.weekly_credits AS x, d.deploys AS y\n FROM _kiro_weekly k INNER JOIN _deploy_weekly d ON k.week_start = d.week_start\n),\n_stats AS (\n SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined\n)\nSELECT CASE\n WHEN s.n < 4 THEN NULL\n WHEN s.sx = 0 OR s.sy = 0 THEN 0\n ELSE ROUND((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined j) / (s.n * s.sx * s.sy), 2)\nEND AS 'r'\nFROM _stats s", + "refId": "A" + } + ], + "title": "Credits vs Deploy Freq (r)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly credits overlaid with deployment count", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 18, "x": 6, "y": 20 }, + "id": 22, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(deploys) AS 'Deployments'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS deploys\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY) AS time,\n 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY DATE_SUB(DATE(cdc.finished_date), INTERVAL WEEKDAY(DATE(cdc.finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Credits vs Deployments", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 28 }, + "id": 30, + "panels": [], + "title": "Change Failure Rate", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Change failure rate: % of deployments that caused incidents", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 29 }, + "id": 31, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(credits) AS 'Credits Used', SUM(cfr) AS 'Change Failure Rate'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(credits_used) AS credits, 0 AS cfr\n FROM _tool_q_dev_user_report WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY) AS time,\n 0 AS credits,\n SUM(CASE WHEN i.id IS NOT NULL THEN 1 ELSE 0 END) / COUNT(DISTINCT d.deployment_id) AS cfr\n FROM (\n SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date\n FROM cicd_deployment_commits cdc\n JOIN project_mapping pm ON cdc.cicd_scope_id = pm.row_id AND pm.`table` = 'cicd_scopes'\n WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION'\n AND $__timeFilter(cdc.finished_date)\n GROUP BY cdc.cicd_deployment_id\n ) d\n LEFT JOIN project_incident_deployment_relationships pidr ON d.deployment_id = pidr.deployment_id\n LEFT JOIN incidents i ON pidr.id = i.id\n GROUP BY DATE_SUB(DATE(d.deployment_finished_date), INTERVAL WEEKDAY(DATE(d.deployment_finished_date)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Credits vs Change Failure Rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "dora", "kiro", "correlation"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Credits + DORA Correlation", + "uid": "kiro_credits_dora", + "version": 1 +} diff --git a/grafana/dashboards/mysql/language-ai-heatmap.json b/grafana/dashboards/mysql/language-ai-heatmap.json new file mode 100644 index 00000000000..789b291e5c1 --- /dev/null +++ b/grafana/dashboards/mysql/language-ai-heatmap.json @@ -0,0 +1,156 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Completion requests, avg context size, and completion rate per language", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { "cellHeight": "sm", "showHeader": true, "sortBy": [{ "desc": true, "displayName": "Requests" }] }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS 'Language',\n COUNT(*) AS 'Requests',\n ROUND(AVG(completions_count), 2) AS 'Avg Completions',\n ROUND(AVG(left_context_length)) AS 'Avg Left Context',\n ROUND(AVG(right_context_length)) AS 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) AS 'Avg Total Context',\n COUNT(DISTINCT user_id) AS 'Users'\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Language Completion Profile", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily completion requests by language", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", "fillOpacity": 80, "lineWidth": 1, + "stacking": { "mode": "normal" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 9 }, + "id": 2, + "options": { + "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT DATE(timestamp) AS time,\n CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric,\n COUNT(*) AS value\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp), file_extension\nORDER BY time", + "refId": "A" + } + ], + "title": "Daily Completions by Language", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Which file types are most active during chat sessions", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 17 }, + "id": 3, + "options": { + "displayLabels": ["name", "percent"], + "legend": { "displayMode": "table", "placement": "right", "showLegend": true, "values": ["value", "percent"] }, + "pieType": "donut", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": true }, + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file)' ELSE active_file_extension END AS 'File Type',\n COUNT(*) AS 'Chat Events'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 10", + "refId": "A" + } + ], + "title": "Active File Types During Chat", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Context size trends for top languages", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 17 }, + "id": 4, + "options": { + "legend": { "calcs": ["mean"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT DATE(timestamp) AS time,\n file_extension AS metric,\n ROUND(AVG(left_context_length + right_context_length)) AS value\nFROM _tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\n AND file_extension IN (SELECT file_extension FROM _tool_q_dev_completion_log GROUP BY file_extension ORDER BY COUNT(*) DESC LIMIT 5)\nGROUP BY DATE(timestamp), file_extension\nORDER BY time", + "refId": "A" + } + ], + "title": "Avg Context Size by Language (Top 5)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "kiro", "language", "completions"], + "templating": { "list": [] }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Language AI Heatmap", + "uid": "kiro_language_heatmap", + "version": 1 +} diff --git a/grafana/dashboards/mysql/multi-ai-comparison.json b/grafana/dashboards/mysql/multi-ai-comparison.json new file mode 100644 index 00000000000..617cda77637 --- /dev/null +++ b/grafana/dashboards/mysql/multi-ai-comparison.json @@ -0,0 +1,299 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage", + "type": "link", + "url": "/d/qdev_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Copilot Adoption", + "type": "link", + "url": "/d/copilot_adoption" + } + ], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro (Amazon Q Developer) adoption and usage side by side.\n\n**Note:** Copilot metrics come from `_tool_copilot_enterprise_daily_metrics` (enterprise-level aggregates). Kiro metrics come from `_tool_q_dev_user_report` and `_tool_q_dev_user_data`. Select the Copilot connection and scope using the dropdowns above.", + "mode": "markdown" + }, + "title": "About", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "id": 2, + "panels": [], + "title": "Active Users", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly active users for both tools", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 4 }, + "id": 3, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n COUNT(DISTINCT user_id) AS 'Kiro Active Users'\nFROM _tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "refId": "A" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n MAX(daily_active_users) AS 'Copilot Active Users'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time", + "refId": "B" + } + ], + "title": "Weekly Active Users Comparison", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, + "id": 10, + "panels": [], + "title": "Code Generation", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly code generation and acceptance events", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 }, + "id": 11, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_suggestions_count) AS 'Kiro: Suggestions',\n SUM(inline_acceptance_count) AS 'Kiro: Accepted'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Kiro: Code Suggestions & Acceptance", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly Copilot code generation and acceptance events", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 13 }, + "id": 12, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n SUM(code_generation_activity_count) AS 'Copilot: Suggestions',\n SUM(code_acceptance_activity_count) AS 'Copilot: Accepted'\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\nGROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Copilot: Code Suggestions & Acceptance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "id": 20, + "panels": [], + "title": "Lines of Code & Acceptance Rate", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly AI-generated lines of code accepted", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisLabel": "", "axisPlacement": "auto", + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "pointSize": 5, "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "id": 21, + "options": { + "legend": { "calcs": ["mean", "sum"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(kiro_loc) AS 'Kiro LOC Accepted', SUM(copilot_loc) AS 'Copilot LOC Added'\nFROM (\n SELECT DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY) AS time,\n SUM(inline_ai_code_lines + chat_ai_code_lines) AS kiro_loc, 0 AS copilot_loc\n FROM _tool_q_dev_user_data WHERE $__timeFilter(date)\n GROUP BY DATE_SUB(DATE(date), INTERVAL WEEKDAY(DATE(date)) DAY)\n UNION ALL\n SELECT DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY) AS time,\n 0 AS kiro_loc, SUM(loc_added_sum) AS copilot_loc\n FROM _tool_copilot_enterprise_daily_metrics\n WHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)\n GROUP BY DATE_SUB(DATE(day), INTERVAL WEEKDAY(DATE(day)) DAY)\n) combined\nGROUP BY time ORDER BY time", + "refId": "A" + } + ], + "title": "LOC Accepted: Kiro vs Copilot", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Overall acceptance rates for both tools", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "id": 22, + "options": { + "barRadius": 0.1, + "barWidth": 0.5, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'Kiro' AS Tool,\n ROUND(SUM(inline_acceptance_count) * 100.0 / NULLIF(SUM(inline_suggestions_count), 0), 1) AS 'Acceptance Rate'\nFROM _tool_q_dev_user_data\nWHERE $__timeFilter(date)\nUNION ALL\nSELECT 'Copilot' AS Tool,\n ROUND(SUM(code_acceptance_activity_count) * 100.0 / NULLIF(SUM(code_generation_activity_count), 0), 1)\nFROM _tool_copilot_enterprise_daily_metrics\nWHERE connection_id = ${copilot_connection_id}\n AND scope_id = '${copilot_scope_id}'\n AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Acceptance Rate Comparison", + "type": "bargauge" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "copilot", "kiro", "comparison"], + "templating": { + "list": [ + { + "current": {}, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "hide": 0, + "label": "Copilot Connection", + "name": "copilot_connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "refresh": 1, + "type": "query" + }, + { + "current": {}, + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)", + "hide": 0, + "label": "Copilot Scope", + "name": "copilot_scope_id", + "options": [], + "query": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS UNSIGNED)", + "refresh": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Multi-AI Tool Comparison", + "uid": "multi_ai_comparison", + "version": 1 +} diff --git a/grafana/dashboards/PagerDuty.json b/grafana/dashboards/mysql/pager-duty.json similarity index 95% rename from grafana/dashboards/PagerDuty.json rename to grafana/dashboards/mysql/pager-duty.json index fcf63114531..49101c42021 100644 --- a/grafana/dashboards/PagerDuty.json +++ b/grafana/dashboards/mysql/pager-duty.json @@ -77,7 +77,7 @@ "content": "- Use Cases: This dashboard shows the incident data from PagerDuty.\n- Data Source Required: PagerDuty", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -116,7 +116,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -165,10 +165,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -218,7 +218,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -268,10 +268,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -321,7 +321,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", "fieldConfig": { "defaults": { @@ -388,10 +388,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -441,7 +441,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -495,10 +495,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -548,7 +548,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -629,10 +629,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -708,7 +708,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -764,10 +764,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -818,7 +818,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -873,10 +873,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -927,7 +927,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1012,10 +1012,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1070,7 +1070,7 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "1. The cumulative distribution of MTTR\n2. Each point refers to the percent rank of a distinct duration to resolve incidents.", "fill": 0, "fillGradient": 4, @@ -1107,7 +1107,7 @@ "alertThreshold": false }, "percentage": false, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "pointradius": 0.5, "points": false, "renderer": "flot", @@ -1117,7 +1117,7 @@ "steppedLine": false, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -1232,7 +1232,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -1265,7 +1265,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from boards where id like 'pagerduty%'", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/QDevDORA.json b/grafana/dashboards/mysql/q-dev-dora.json similarity index 94% rename from grafana/dashboards/QDevDORA.json rename to grafana/dashboards/mysql/q-dev-dora.json index f132872be63..3d584dfd1fb 100644 --- a/grafana/dashboards/QDevDORA.json +++ b/grafana/dashboards/mysql/q-dev-dora.json @@ -64,7 +64,7 @@ "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "title": "Dashboard Introduction", "type": "text" }, @@ -82,7 +82,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of unique users who used Q Dev AI features", "fieldConfig": { "defaults": { @@ -123,10 +123,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -138,7 +138,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Total AI-generated code lines accepted (Inline + Chat)", "fieldConfig": { "defaults": { @@ -180,10 +180,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -195,7 +195,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Acceptance rate of inline AI suggestions", "fieldConfig": { "defaults": { @@ -245,10 +245,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -260,7 +260,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Number of production deployments in selected period", "fieldConfig": { "defaults": { @@ -301,10 +301,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -316,7 +316,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Median lead time for changes in hours", "fieldConfig": { "defaults": { @@ -366,10 +366,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -381,7 +381,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Percentage of deployments that caused incidents", "fieldConfig": { "defaults": { @@ -431,10 +431,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -459,7 +459,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", "fieldConfig": { "defaults": { @@ -579,10 +579,10 @@ "sort": "none" } }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -594,7 +594,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", "fieldConfig": { "defaults": { @@ -718,10 +718,10 @@ "sort": "none" } }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -733,7 +733,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", "fieldConfig": { "defaults": { @@ -857,10 +857,10 @@ "sort": "none" } }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -872,7 +872,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", "fieldConfig": { "defaults": { @@ -992,10 +992,10 @@ "sort": "none" } }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1020,7 +1020,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", "fieldConfig": { "defaults": { @@ -1142,10 +1142,10 @@ } ] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1174,7 +1174,7 @@ "text": ["All"], "value": ["$__all"] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "SELECT DISTINCT name FROM projects", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/mysql/qdev_executive.json b/grafana/dashboards/mysql/qdev_executive.json new file mode 100644 index 00000000000..fffbc99ba60 --- /dev/null +++ b/grafana/dashboards/mysql/qdev_executive.json @@ -0,0 +1,768 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Usage (New)", + "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", + "type": "link", + "url": "/d/qdev_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Feature Metrics (Legacy)", + "tooltip": "Kiro Legacy Feature Metrics (old format)", + "type": "link", + "url": "/d/qdev_feature_metrics" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Prompt Logging", + "tooltip": "Kiro AI Activity Insights - Prompt Logging", + "type": "link", + "url": "/d/qdev_logging" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "KPI Overview (cross-source)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) as 'WAU'\nFROM lake._tool_q_dev_chat_log\nWHERE timestamp >= DATE_SUB(NOW(), INTERVAL 7 DAY)", + "refId": "A" + } + ], + "title": "Weekly Active Users (logging)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(r.credits_used) / NULLIF(SUM(d.total_accepted), 0), 2) as 'Credits per Accepted Line'\nFROM (\n SELECT user_id, date, SUM(credits_used) as credits_used\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r\nJOIN (\n SELECT user_id, date,\n (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) as total_accepted\n FROM lake._tool_q_dev_user_data\n WHERE $__timeFilter(date)\n) d ON r.user_id = d.user_id AND r.date = d.date", + "refId": "A" + } + ], + "title": "Credits Efficiency (new + legacy)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1) as 'Acceptance %'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Inline Acceptance Rate (legacy)", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Percentage of users who used steering rules (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) / NULLIF(COUNT(DISTINCT user_id), 0) * 100, 0) as 'Steering %'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Adoption (logging)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "panels": [], + "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly active user count over time (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n COUNT(DISTINCT user_id) as 'Active Users'\nFROM (\n SELECT user_id, YEARWEEK(timestamp, 1) as yw\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n) t\nGROUP BY yw\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Active Users Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "New vs returning users by week (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n STR_TO_DATE(CONCAT(yw, ' Monday'), '%X%V %W') as time,\n SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) as 'New Users',\n SUM(CASE WHEN yw != first_yw THEN 1 ELSE 0 END) as 'Returning Users'\nFROM (\n SELECT DISTINCT u.user_id, YEARWEEK(u.timestamp, 1) as yw, f.first_yw\n FROM lake._tool_q_dev_chat_log u\n JOIN (SELECT user_id, YEARWEEK(MIN(timestamp), 1) as first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) f\n ON u.user_id = f.user_id\n WHERE $__timeFilter(u.timestamp)\n) weekly\nGROUP BY yw\nORDER BY time", + "refId": "A" + } + ], + "title": "New vs Returning Users (Weekly)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 102, + "panels": [], + "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Cumulative credits this month vs projected total (from new user_report)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(SUM(credits_used)) OVER (ORDER BY date) as 'Cumulative Credits',\n (SELECT SUM(credits_used) / COUNT(DISTINCT date) * DAY(LAST_DAY(CURDATE()))\n FROM lake._tool_q_dev_user_report\n WHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())) as 'Projected Monthly'\nFROM lake._tool_q_dev_user_report\nWHERE YEAR(date) = YEAR(CURDATE()) AND MONTH(date) = MONTH(CURDATE())\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Credits Pace vs Projected (This Month)", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Power tier users with no activity in the last 14 days (from new user_report)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n MAX(subscription_tier) as 'Tier',\n ROUND(SUM(credits_used), 1) as 'Total Credits Used',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier = 'POWER'\nGROUP BY user_id\nHAVING MAX(date) < DATE_SUB(NOW(), INTERVAL 14 DAY)\nORDER BY MAX(date)", + "refId": "A" + } + ], + "title": "Idle Power Users (No Activity in 14 Days)", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 103, + "panels": [], + "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(d.display_name), d.user_id) as 'User',\n COALESCE(MAX(r.subscription_tier), '') as 'Tier',\n ROUND(SUM(r.credits_used), 1) as 'Credits Used',\n SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) as 'Total Accepted Lines',\n CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0\n THEN ROUND(SUM(r.credits_used) / SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 2)\n ELSE NULL END as 'Credits/Line',\n CONCAT(ROUND(SUM(d.inline_acceptance_count) / NULLIF(SUM(d.inline_suggestions_count), 0) * 100, 1), '%') as 'Accept Rate',\n SUM(d.code_review_findings_count) as 'Review Findings',\n SUM(d.test_generation_event_count) as 'Test Gen Events',\n SUM(d.dev_accepted_lines) as 'Agentic Lines',\n MIN(d.date) as 'First Active',\n MAX(d.date) as 'Last Active'\nFROM lake._tool_q_dev_user_data d\nLEFT JOIN (\n SELECT user_id, date, SUM(credits_used) as credits_used, MAX(subscription_tier) as subscription_tier\n FROM lake._tool_q_dev_user_report\n WHERE $__timeFilter(date)\n GROUP BY user_id, date\n) r ON d.user_id = r.user_id AND d.date = r.date\nWHERE $__timeFilter(d.date)\nGROUP BY d.user_id\nORDER BY SUM(r.credits_used) DESC", + "refId": "A" + } + ], + "title": "User Productivity & Efficiency", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "executive", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Executive Dashboard", + "uid": "qdev_executive", + "version": 1 +} diff --git a/grafana/dashboards/mysql/qdev_feature_metrics.json b/grafana/dashboards/mysql/qdev_feature_metrics.json new file mode 100644 index 00000000000..5156e652a34 --- /dev/null +++ b/grafana/dashboards/mysql/qdev_feature_metrics.json @@ -0,0 +1,948 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(inline_suggestions_count) as 'Inline Suggestions',\n SUM(inline_acceptance_count) as 'Inline Accepted',\n SUM(chat_messages_sent) as 'Chat Messages',\n SUM(chat_ai_code_lines) as 'Chat AI Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Agentic Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Legacy Feature Metrics Overview", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 20, + "panels": [], + "title": "Inline Suggestions", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily inline suggestion and acceptance counts", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n SUM(inline_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Inline Suggestions & Acceptance", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) as 'Inline Suggestions',\n SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) as 'Code Fix',\n SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) as 'Inline Chat'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Acceptance Rate Trends", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 21, + "panels": [], + "title": "Chat & Agentic (Dev)", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily chat messages sent and AI-generated code lines from chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(chat_messages_sent) as 'Messages Sent',\n SUM(chat_messages_interacted) as 'Messages Interacted',\n SUM(chat_ai_code_lines) as 'AI Code Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Chat Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Agentic (Dev) code generation and acceptance metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Generation Events',\n SUM(dev_generated_lines) as 'Generated Lines',\n SUM(dev_accepted_lines) as 'Accepted Lines',\n SUM(dev_acceptance_event_count) as 'Acceptance Events'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Agentic (Dev) Activity", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "title": "Code Review, Test Gen & Transformations", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Code review findings and test generation metrics over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(code_review_succeeded_event_count) as 'Reviews Succeeded',\n SUM(code_review_failed_event_count) as 'Reviews Failed'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Code Review Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Test generation events and acceptance over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(test_generation_generated_tests) as 'Tests Generated',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(test_generation_generated_lines) as 'Lines Generated',\n SUM(test_generation_accepted_lines) as 'Lines Accepted'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Test Generation Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Doc generation and code transformation events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(doc_generation_accepted_line_additions) as 'Doc Lines Accepted',\n SUM(transformation_event_count) as 'Transformation Events',\n SUM(transformation_lines_generated) as 'Transform Lines Generated'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Doc Generation & Transformations", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Number of users who used each feature in the selected period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 9, + "options": { + "barRadius": 0.1, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'Chat' as Feature, COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) as Users FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)\nUNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Feature Adoption (Users per Feature)", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [], + "title": "Per-User Detail", + "type": "row" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Per-user breakdown of legacy feature-level metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(inline_suggestions_count) as 'Suggestions',\n SUM(inline_acceptance_count) as 'Accepted',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 1), '%') as 'Accept %',\n SUM(chat_messages_sent) as 'Chat Msgs',\n SUM(chat_ai_code_lines) as 'Chat Lines',\n SUM(dev_accepted_lines) as 'Agentic Lines',\n SUM(code_review_findings_count) as 'Review Findings',\n SUM(test_generation_accepted_tests) as 'Tests Accepted',\n SUM(doc_generation_event_count) as 'Doc Gen',\n SUM(transformation_event_count) as 'Transforms',\n MIN(date) as 'First Active',\n MAX(date) as 'Last Active'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_suggestions_count) DESC", + "refId": "A" + } + ], + "title": "Per-User Feature Metrics", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "legacy", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Legacy Feature Metrics", + "uid": "qdev_feature_metrics", + "version": 1 +} diff --git a/grafana/dashboards/mysql/qdev_logging.json b/grafana/dashboards/mysql/qdev_logging.json new file mode 100644 index 00000000000..aa7826ae243 --- /dev/null +++ b/grafana/dashboards/mysql/qdev_logging.json @@ -0,0 +1,1118 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Overview of logging event metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Chat Users',\n (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id != '') as 'Conversations',\n (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Events',\n (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) as 'Completion Users',\n (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Code References',\n (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) as 'Web Links Cited'", + "refId": "A" + } + ], + "title": "Logging Overview", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Hourly distribution of AI usage activity (chat + completions)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hour of Day", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n LPAD(CAST(hour_of_day AS CHAR), 2, '0') as 'Hour',\n SUM(chat_count) as 'Chat Events',\n SUM(completion_count) as 'Completion Events'\nFROM (\n SELECT HOUR(timestamp) as hour_of_day, COUNT(*) as chat_count, 0 as completion_count\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n UNION ALL\n SELECT HOUR(timestamp) as hour_of_day, 0 as chat_count, COUNT(*) as completion_count\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY HOUR(timestamp)\n) combined\nGROUP BY hour_of_day\nORDER BY hour_of_day", + "refId": "A" + } + ], + "title": "Active Hours Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 11, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)'\n ELSE chat_trigger_type\n END as 'Trigger Type',\n COUNT(*) as 'Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY chat_trigger_type\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Chat Trigger Type Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of model usage across chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 3, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Requests$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN model_id = '' OR model_id IS NULL THEN '(unknown)'\n ELSE model_id\n END as 'Model',\n COUNT(*) as 'Requests'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY model_id\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Model Usage Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Top file extensions used with inline completions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN file_extension = '' THEN '(unknown)'\n ELSE file_extension\n END as 'File Type',\n COUNT(*) as 'Completions'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "File Type Usage (Completions)", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average number of chat events per conversation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n COUNT(*) / NULLIF(COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END), 0) as 'Avg Turns per Conversation',\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as 'Unique Conversations',\n COUNT(*) as 'Total Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Conversation Depth Analysis", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily chat and completion events over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(chat) as 'Chat Events', SUM(completions) as 'Completion Events'\nFROM (\n SELECT DATE(timestamp) as time, COUNT(*) as chat, 0 as completions\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n UNION ALL\n SELECT DATE(timestamp) as time, 0 as chat, COUNT(*) as completions\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY DATE(timestamp)\n) combined\nGROUP BY time\nORDER BY time", + "refId": "A" + } + ], + "title": "Daily Event Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Per-user logging activity summary", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(u.display_name, u.user_id) as 'User',\n u.user_id as 'User ID',\n u.chat_events as 'Chat Events',\n u.conversations as 'Conversations',\n ROUND(u.chat_events / NULLIF(u.conversations, 0), 1) as 'Avg Turns',\n COALESCE(c.completion_events, 0) as 'Completion Events',\n COALESCE(c.files_count, 0) as 'Distinct Files',\n ROUND(u.avg_prompt_len) as 'Avg Prompt Len',\n ROUND(u.avg_response_len) as 'Avg Response Len',\n u.steering_count as 'Steering Uses',\n u.spec_count as 'Spec Mode Uses',\n u.code_ref_count as 'Code Refs',\n u.web_link_count as 'Web Links',\n u.models_used as 'Models Used',\n u.first_seen as 'First Seen',\n GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) as 'Last Seen'\nFROM (\n SELECT\n user_id,\n MAX(display_name) as display_name,\n COUNT(*) as chat_events,\n COUNT(DISTINCT CASE WHEN conversation_id != '' THEN conversation_id END) as conversations,\n AVG(prompt_length) as avg_prompt_len,\n AVG(response_length) as avg_response_len,\n GROUP_CONCAT(DISTINCT CASE WHEN model_id != '' AND model_id IS NOT NULL THEN model_id END ORDER BY model_id SEPARATOR ', ') as models_used,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as steering_count,\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as spec_count,\n SUM(code_reference_count) as code_ref_count,\n SUM(web_link_count) as web_link_count,\n MIN(timestamp) as first_seen,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_chat_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) u\nLEFT JOIN (\n SELECT\n user_id,\n COUNT(*) as completion_events,\n COUNT(DISTINCT file_name) as files_count,\n MAX(timestamp) as last_seen\n FROM lake._tool_q_dev_completion_log\n WHERE $__timeFilter(timestamp)\n GROUP BY user_id\n) c ON u.user_id = c.user_id\nORDER BY u.user_id", + "refId": "A" + } + ], + "title": "Per-User Activity", + "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 48 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) as 'Using Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) as 'Using Spec Mode',\n SUM(CASE WHEN has_steering = 0 AND is_spec_mode = 0 THEN 1 ELSE 0 END) as 'Plain Chat'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Kiro Feature Adoption", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Top file extensions active during chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 9, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)'\n ELSE active_file_extension\n END as 'File Type',\n COUNT(*) as 'Chat Events'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY active_file_extension\nORDER BY COUNT(*) DESC\nLIMIT 15", + "refId": "A" + } + ], + "title": "Active File Types in Chat", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "How often Kiro responses include code references and web links", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 12, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) as 'With Code References',\n SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) as 'With Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'With Followup Prompts',\n SUM(CASE WHEN code_reference_count = 0 AND web_link_count = 0 AND has_followup_prompts = 0 THEN 1 ELSE 0 END) as 'Plain Response'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Response Enrichment Breakdown", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average and maximum prompt/response lengths over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n AVG(prompt_length) as 'Avg Prompt Length',\n AVG(response_length) as 'Avg Response Length',\n MAX(prompt_length) as 'Max Prompt Length',\n MAX(response_length) as 'Max Response Length'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Prompt & Response Length Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Average code context size provided to inline completions over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Characters", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n ROUND(AVG(left_context_length)) as 'Avg Left Context',\n ROUND(AVG(right_context_length)) as 'Avg Right Context',\n ROUND(AVG(left_context_length + right_context_length)) as 'Avg Total Context'\nFROM lake._tool_q_dev_completion_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Completion Context Size Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily trend of code references and web links in chat responses", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(timestamp) as time,\n SUM(code_reference_count) as 'Code References',\n SUM(web_link_count) as 'Web Links',\n SUM(CASE WHEN has_followup_prompts = 1 THEN 1 ELSE 0 END) as 'Followup Prompts'\nFROM lake._tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE(timestamp)\nORDER BY DATE(timestamp)", + "refId": "A" + } + ], + "title": "Response Enrichment Trends", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "logging", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro AI Activity Insights", + "uid": "qdev_logging", + "version": 1 +} diff --git a/grafana/dashboards/qdev_user_data.json b/grafana/dashboards/mysql/qdev_user_data.json similarity index 60% rename from grafana/dashboards/qdev_user_data.json rename to grafana/dashboards/mysql/qdev_user_data.json index ff55b9ff81d..affaf57244c 100644 --- a/grafana/dashboards/qdev_user_data.json +++ b/grafana/dashboards/mysql/qdev_user_data.json @@ -19,7 +19,7 @@ "links": [], "panels": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Overview of key user metrics", "fieldConfig": { "defaults": { @@ -63,10 +63,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -115,7 +115,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Daily AI code line changes across all users", "fieldConfig": { "defaults": { @@ -192,10 +192,10 @@ "sort": "none" } }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -244,7 +244,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Daily AI interaction trends across all users", "fieldConfig": { "defaults": { @@ -321,10 +321,10 @@ "sort": "none" } }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -373,7 +373,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Code review metrics over time", "fieldConfig": { "defaults": { @@ -450,10 +450,10 @@ "sort": "none" } }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "time_series", "group": [], "metricColumn": "none", @@ -501,7 +501,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Daily acceptance rate of AI suggestions", "fieldConfig": { "defaults": { @@ -578,10 +578,10 @@ "sort": "none" } }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "time_series", "group": [], @@ -630,7 +630,7 @@ "type": "timeseries" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "User AI interaction metrics", "fieldConfig": { "defaults": { @@ -721,16 +721,16 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.6.2", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n COALESCE(display_name, user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id, display_name\nORDER BY SUM(inline_ai_code_lines) DESC", + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n SUM(chat_ai_code_lines) as 'Accepted Lines (Chat)',\n SUM(transformation_lines_ingested) as 'Lines Ingested (Java Transform)',\n SUM(transformation_lines_generated) as 'Lines Generated (Java Transform)',\n SUM(transformation_event_count) as 'Event Count (Java Transform)',\n SUM(code_review_findings_count) as 'Findings (Code Review)',\n SUM(code_fix_accepted_lines) as 'Accepted Lines (Code Fix)',\n SUM(code_fix_generated_lines) as 'Generated Lines (Code Fix)',\n SUM(code_fix_acceptance_event_count) as 'Accepted Count (Code Fix)',\n SUM(code_fix_generation_event_count) as 'Generated Count (Code Fix)',\n CONCAT(ROUND(SUM(code_fix_acceptance_event_count) / NULLIF(SUM(code_fix_generation_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Code Fix)',\n SUM(inline_ai_code_lines) as 'Accepted Lines (Inline Suggestion)',\n SUM(inline_acceptance_count) as 'Accepted Count (Inline Suggestion)',\n SUM(inline_suggestions_count) as 'Total Count (Inline Suggestion)',\n CONCAT(ROUND(SUM(inline_acceptance_count) / NULLIF(SUM(inline_suggestions_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Suggestion)',\n SUM(inline_chat_accepted_line_additions) as 'Accepted Line Additions (Inline Chat)',\n SUM(inline_chat_accepted_line_deletions) as 'Accepted Line Deletions (Inline Chat)',\n SUM(inline_chat_acceptance_event_count) as 'Accepted Events (Inline Chat)',\n SUM(inline_chat_total_event_count) as 'Total Events (Inline Chat)',\n CONCAT(ROUND(SUM(inline_chat_acceptance_event_count) / NULLIF(SUM(inline_chat_total_event_count), 0) * 100, 2), '%') as 'Acceptance Rate (Inline Chat)',\n SUM(doc_generation_event_count) as 'Doc Gen Events',\n SUM(test_generation_event_count) as 'Test Gen Events',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY user_id\nORDER BY SUM(inline_ai_code_lines) DESC", "refId": "A", "select": [ [ @@ -771,6 +771,393 @@ ], "title": "User Interactions", "type": "table" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily doc generation events and accepted/rejected lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(doc_generation_event_count) as 'Doc Generation Events',\n SUM(doc_generation_accepted_line_additions) as 'Accepted Line Additions',\n SUM(doc_generation_accepted_line_updates) as 'Accepted Line Updates',\n SUM(doc_generation_rejected_line_additions) as 'Rejected Line Additions',\n SUM(doc_generation_rejected_line_updates) as 'Rejected Line Updates'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Doc Generation Metrics", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily test generation events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(test_generation_event_count) as 'Test Generation Events',\n SUM(test_generation_accepted_tests) as 'Accepted Tests',\n SUM(test_generation_generated_tests) as 'Generated Tests',\n SUM(test_generation_accepted_lines) as 'Accepted Lines',\n SUM(test_generation_generated_lines) as 'Generated Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Test Generation Metrics", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily agentic dev events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(dev_generation_event_count) as 'Dev Generation Events',\n SUM(dev_acceptance_event_count) as 'Dev Acceptance Events',\n SUM(dev_generated_lines) as 'Dev Generated Lines',\n SUM(dev_accepted_lines) as 'Dev Accepted Lines'\nFROM lake._tool_q_dev_user_data\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Dev (Agentic) Metrics", + "type": "timeseries" } ], "preload": false, @@ -789,7 +1176,7 @@ }, "timepicker": {}, "timezone": "utc", - "title": "Q Dev User Data Dashboard", + "title": "Kiro Code Metrics Dashboard", "uid": "qdev_user_data", "version": 1 } \ No newline at end of file diff --git a/grafana/dashboards/mysql/qdev_user_report.json b/grafana/dashboards/mysql/qdev_user_report.json new file mode 100644 index 00000000000..acd0f14b3b6 --- /dev/null +++ b/grafana/dashboards/mysql/qdev_user_report.json @@ -0,0 +1,464 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 101, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Overview of credits and usage metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n SUM(credits_used) as 'Total Credits Used',\n COUNT(DISTINCT user_id) as 'Active Users',\n SUM(total_messages) as 'Total Messages',\n SUM(chat_conversations) as 'Total Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Overview Stats", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily credits consumed broken down by subscription tier", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n subscription_tier as metric,\n SUM(credits_used) as value\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date, subscription_tier\nORDER BY date", + "refId": "A" + } + ], + "title": "Daily Credits Consumed by Subscription Tier", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Daily messages and conversations broken down by client type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date as time,\n SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) as 'Messages (IDE)',\n SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) as 'Messages (CLI)',\n SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) as 'Messages (Plugin)',\n SUM(chat_conversations) as 'Conversations'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY date\nORDER BY date", + "refId": "A" + } + ], + "title": "Daily Messages & Conversations", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Distribution of users across subscription tiers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n subscription_tier as 'Tier',\n COUNT(DISTINCT user_id) as 'Users'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\n AND subscription_tier IS NOT NULL\n AND subscription_tier != ''\nGROUP BY subscription_tier\nORDER BY COUNT(DISTINCT user_id) DESC", + "refId": "A" + } + ], + "title": "Subscription Tier Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Per-user credits, messages, and subscription details", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Credits Used" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Overage" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Yes": { + "color": "orange", + "index": 0 + }, + "No": { + "color": "green", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) as 'User',\n subscription_tier as 'Tier',\n client_type as 'Client',\n SUM(credits_used) as 'Credits Used',\n SUM(total_messages) as 'Messages',\n SUM(chat_conversations) as 'Conversations',\n SUM(overage_credits_used) as 'Overage Credits',\n CASE WHEN MAX(CAST(overage_enabled AS UNSIGNED)) = 1 THEN 'Yes' ELSE 'No' END as 'Overage',\n MIN(date) as 'First Activity',\n MAX(date) as 'Last Activity'\nFROM lake._tool_q_dev_user_report\nWHERE $__timeFilter(date)\nGROUP BY user_id, subscription_tier, client_type\nORDER BY user_id DESC", + "refId": "A" + } + ], + "title": "Per-User Credits & Activity", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "user_report", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Usage Dashboard", + "uid": "qdev_user_report", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/SonarQubeCloud.json b/grafana/dashboards/mysql/sonar-qube-cloud.json similarity index 77% rename from grafana/dashboards/SonarQubeCloud.json rename to grafana/dashboards/mysql/sonar-qube-cloud.json index 1a290703897..064285ab9eb 100644 --- a/grafana/dashboards/SonarQubeCloud.json +++ b/grafana/dashboards/mysql/sonar-qube-cloud.json @@ -47,10 +47,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard shows the code quality metrics from SonarCloud.\n- Data Source Required: SonarCloud\n- This dashboard does not honor the time filter on the top-right side as SonarQube metrics are all from the latest scan.", + "content": "- Use Cases: This dashboard shows the code quality metrics from SonarCloud.\n- Data Source Required: SonarCloud\n- Snapshot panels (Software Quality, Security Review, Test, etc.) show the latest scan and do not honor the time filter.\n- **Historical Trends** panels at the bottom respond to the time range selector and show metric history over time.", "mode": "markdown" }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -89,7 +89,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -144,11 +144,11 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -177,7 +177,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -232,11 +232,11 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -265,7 +265,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -320,11 +320,11 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -366,7 +366,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -422,11 +422,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -455,7 +455,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -511,11 +511,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -570,7 +570,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -626,11 +626,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -659,7 +659,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -716,11 +716,11 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -749,7 +749,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -806,11 +806,11 @@ "textMode": "value", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -865,7 +865,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -921,11 +921,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -954,7 +954,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1010,11 +1010,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1056,7 +1056,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1113,11 +1113,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1146,7 +1146,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1203,11 +1203,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1236,7 +1236,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1293,11 +1293,11 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1339,7 +1339,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1397,11 +1397,11 @@ }, "showHeader": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "13.0.2", "targets": [ { "dataset": "lake", - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1428,6 +1428,263 @@ ], "title": "Code Quality Metrics by Files (Top 20 order by Bugs)", "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 27, + "panels": [ + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 28, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.coverage AS 'Coverage %'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Coverage Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 29, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.bugs AS 'Bugs',\n pmh.vulnerabilities AS 'Vulnerabilities'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Bugs & Vulnerabilities Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 30, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.code_smells AS 'Code Smells'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Code Smells Trend", + "type": "timeseries" + }, + { + "datasource": "mysql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 31, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "mysql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n pmh.analysis_date AS time,\n pmh.duplicated_lines_density AS 'Duplication %'\nFROM cq_project_metrics_history pmh\nWHERE\n pmh.project_key in (${project_id})\n AND $__timeFilter(pmh.analysis_date)\nORDER BY pmh.analysis_date ASC", + "refId": "A" + } + ], + "title": "Duplication Trend", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Historical Trends", + "type": "row" } ], "refresh": "", @@ -1448,7 +1705,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) as text from cq_projects", "hide": 0, "includeAll": true, @@ -1473,7 +1730,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct severity from cq_issues where id like 'sonar%'", "hide": 0, "includeAll": true, @@ -1491,7 +1748,7 @@ ] }, "time": { - "from": "now", + "from": "now-6M", "to": "now" }, "timepicker": {}, diff --git a/grafana/dashboards/mysql/steering-adoption-tracker.json b/grafana/dashboards/mysql/steering-adoption-tracker.json new file mode 100644 index 00000000000..14711cfd609 --- /dev/null +++ b/grafana/dashboards/mysql/steering-adoption-tracker.json @@ -0,0 +1,212 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN has_steering = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Steering", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "purple" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(COUNT(DISTINCT CASE WHEN is_spec_mode = 1 THEN user_id END) * 100.0 / NULLIF(COUNT(DISTINCT user_id), 0), 0) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Spec Mode", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", "graphMode": "area", + "reduceOptions": { "calcs": ["sum"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'value'\nFROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Requests with Steering", + "type": "stat" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Weekly trend of steering and spec mode adoption rate", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", "fillOpacity": 10, "lineInterpolation": "smooth", "lineWidth": 2, + "showPoints": "never", "spanNulls": true, + "stacking": { "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, + "id": 4, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, + "tooltip": { "mode": "multi" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY) AS time,\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Steering Rate',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) / COUNT(*) AS 'Spec Mode Rate'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY DATE_SUB(DATE(timestamp), INTERVAL WEEKDAY(DATE(timestamp)) DAY)\nORDER BY time", + "refId": "A" + } + ], + "title": "Weekly Steering & Spec Mode Adoption Rate", + "type": "timeseries" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Compare prompt and response quality between steering and non-steering sessions", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, + "id": 5, + "options": { + "barRadius": 0.1, "barWidth": 0.5, + "orientation": "horizontal", "showValue": "auto", "stacking": "none", + "tooltip": { "mode": "single" } + }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'With Steering' AS 'Mode', ROUND(AVG(prompt_length)) AS 'Avg Prompt', ROUND(AVG(response_length)) AS 'Avg Response'\nFROM _tool_q_dev_chat_log WHERE has_steering = 1 AND $__timeFilter(timestamp)\nUNION ALL\nSELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length))\nFROM _tool_q_dev_chat_log WHERE has_steering = 0 AND $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Impact: Prompt & Response Length", + "type": "bargauge" + }, + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "description": "Per-user steering and spec mode usage", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, + "id": 6, + "options": { "cellHeight": "sm", "showHeader": true }, + "targets": [ + { + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n COALESCE(MAX(display_name), user_id) AS 'User',\n COUNT(*) AS 'Total Chats',\n SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) AS 'Steering',\n SUM(CASE WHEN is_spec_mode = 1 THEN 1 ELSE 0 END) AS 'Spec Mode',\n ROUND(SUM(CASE WHEN has_steering = 1 THEN 1 ELSE 0 END) * 100.0 / COUNT(*), 1) AS 'Steering %'\nFROM _tool_q_dev_chat_log\nWHERE $__timeFilter(timestamp)\nGROUP BY user_id\nORDER BY COUNT(*) DESC", + "refId": "A" + } + ], + "title": "Per-User Feature Adoption", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": ["q_dev", "kiro", "steering", "adoption"], + "templating": { "list": [] }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Steering & Spec Mode Adoption", + "uid": "kiro_steering_adoption", + "version": 1 +} diff --git a/grafana/dashboards/WeeklyBugRetro.json b/grafana/dashboards/mysql/weekly-bug-retro.json similarity index 95% rename from grafana/dashboards/WeeklyBugRetro.json rename to grafana/dashboards/mysql/weekly-bug-retro.json index 63746d0740d..424e71b18a6 100644 --- a/grafana/dashboards/WeeklyBugRetro.json +++ b/grafana/dashboards/mysql/weekly-bug-retro.json @@ -40,10 +40,10 @@ "showLineNumbers": false, "showMiniMap": false }, - "content": "- Use Cases: This dashboard can be used to track bugs with metrics such as [Bug Age](https://devlake.apache.org/docs/Metrics/BugAge).\n- Data Source Required: GitHub ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional) required) or Jira ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required). Scope config is the settings to define which issues are bugs.", + "content": "- Use Cases: This dashboard can be used to track bugs with metrics such as [Bug Age](https://devlake.apache.org/docs/Metrics/BugAge).\n- Data Source Required: One of the issue tracking tools, e.g. [GitHub](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab) or [Jira](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) (Scope Config required to define which issues are bugs).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -71,7 +71,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -157,7 +157,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -209,7 +209,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -295,7 +295,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -347,7 +347,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -435,7 +435,7 @@ }, "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -500,7 +500,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -554,10 +554,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -609,7 +609,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -666,10 +666,10 @@ "sort": "none" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -721,7 +721,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -825,10 +825,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -880,7 +880,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -934,10 +934,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -989,7 +989,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1048,10 +1048,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1103,7 +1103,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1221,10 +1221,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1276,7 +1276,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1361,10 +1361,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1416,7 +1416,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1470,10 +1470,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1525,7 +1525,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1579,10 +1579,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -1616,7 +1616,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1758,10 +1758,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1813,7 +1813,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1898,10 +1898,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 100 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1953,7 +1953,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -2095,10 +2095,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2170,7 +2170,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/UserManuals/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2198,7 +2198,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '-', id) as text from boards", "hide": 0, "includeAll": true, @@ -2219,7 +2219,7 @@ "text": "BUG", "value": "BUG" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, @@ -2244,7 +2244,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct priority from issues", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/WeeklyCommunityRetro.json b/grafana/dashboards/mysql/weekly-community-retro.json similarity index 94% rename from grafana/dashboards/WeeklyCommunityRetro.json rename to grafana/dashboards/mysql/weekly-community-retro.json index fdf6ed989d3..1dc16a0e0b7 100644 --- a/grafana/dashboards/WeeklyCommunityRetro.json +++ b/grafana/dashboards/mysql/weekly-community-retro.json @@ -44,7 +44,7 @@ "content": "- Use Cases: This dashboard can be used to track community growth by OSS maintainers.\n- Data Source Required: GitHub users' organizations are used to filter issues/PRs created by certain users. [Publicize users' org](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/publicizing-or-hiding-organization-membership) so that Apache DevLake can collect users' org.\n- This dashboard **DOES NOT** honor the time filter on the top-right corner.", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -85,7 +85,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -137,10 +137,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -192,7 +192,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -244,10 +244,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -299,7 +299,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -352,10 +352,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -407,7 +407,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -463,10 +463,10 @@ }, "showHeader": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -500,7 +500,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -557,10 +557,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -594,7 +594,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -726,10 +726,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -781,7 +781,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -837,10 +837,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "metricColumn": "none", @@ -874,7 +874,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1010,10 +1010,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1091,7 +1091,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1143,10 +1143,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1198,7 +1198,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1250,10 +1250,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1305,7 +1305,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1357,10 +1357,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1412,7 +1412,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1464,10 +1464,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1519,7 +1519,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1572,10 +1572,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "format": "table", "group": [], "hide": false, @@ -1609,7 +1609,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "Contributors: the PR authors who have merged PR(s)", "fieldConfig": { "defaults": { @@ -1665,10 +1665,10 @@ }, "showHeader": true }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1720,7 +1720,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1777,10 +1777,10 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1832,7 +1832,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1916,10 +1916,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1971,7 +1971,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2054,10 +2054,10 @@ "xTickLabelRotation": 45, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2109,7 +2109,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2156,10 +2156,10 @@ }, "textMode": "auto" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2211,7 +2211,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2263,10 +2263,10 @@ }, "valueMode": "color" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -2338,7 +2338,7 @@ "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", "mode": "markdown" }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -2370,7 +2370,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '-', id) as text from repos", "hide": 0, "includeAll": true, @@ -2391,7 +2391,7 @@ "text": "All", "value": "$__all" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct type from issues", "hide": 0, "includeAll": true, @@ -2416,7 +2416,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select distinct organization from accounts where organization != ''", "description": "", "hide": 0, diff --git a/grafana/dashboards/WorkLogs.json b/grafana/dashboards/mysql/work-logs.json similarity index 95% rename from grafana/dashboards/WorkLogs.json rename to grafana/dashboards/mysql/work-logs.json index ba2da4761e8..df6d88baa8e 100644 --- a/grafana/dashboards/WorkLogs.json +++ b/grafana/dashboards/mysql/work-logs.json @@ -43,7 +43,7 @@ "content": "- Use Cases: This dashboard shows the work logs of a team or a developer\n- Data Sources Required to show all data: \n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira)\n - You also need to complete the [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard", "mode": "markdown" }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { "datasource": { @@ -69,7 +69,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -209,10 +209,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -241,7 +241,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -352,10 +352,10 @@ "showHeader": true, "sortBy": [] }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -384,7 +384,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -573,10 +573,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -618,7 +618,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -709,10 +709,10 @@ "sort": "none" } }, - "pluginVersion": "9.5.15", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -741,7 +741,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -780,14 +780,14 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "with _accounts as (\n select ua.account_id, ua.user_id, u.name, u.email\n from accounts a \n join user_accounts ua on a.id = ua.account_id\n join users u on ua.user_id = u.id\n where ua.user_id in ($users)\n)\n\nSELECT count(distinct c.sha)\nFROM commits c\njoin _accounts a on c.author_id = a.email\nwhere $__timeFilter(authored_date)\n", + "rawSql": "with _accounts as (\n select ua.account_id, ua.user_id, u.name, a.email\n from accounts a \n join user_accounts ua on a.id = ua.account_id\n join users u on ua.user_id = u.id\n where ua.user_id in ($users)\n)\n\nSELECT count(distinct c.sha)\nFROM commits c\njoin _accounts a on c.author_id = a.email\nwhere $__timeFilter(authored_date)\n", "refId": "A", "sql": { "columns": [ @@ -812,7 +812,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -852,10 +852,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -884,7 +884,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -924,10 +924,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -956,7 +956,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -996,10 +996,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1028,7 +1028,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1111,10 +1111,10 @@ "xTickLabelRotation": 0, "xTickLabelSpacing": 0 }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "group": [], @@ -1179,7 +1179,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1219,10 +1219,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1251,7 +1251,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "fieldConfig": { "defaults": { "color": { @@ -1291,10 +1291,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1323,7 +1323,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "description": "average pr review time from first comment to the last comment", "fieldConfig": { "defaults": { @@ -1364,10 +1364,10 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.0.0", + "pluginVersion": "13.0.2", "targets": [ { - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "editorMode": "code", "format": "table", "rawQuery": true, @@ -1411,7 +1411,7 @@ "text": "None", "value": "" }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(name, '--', id) from teams", "hide": 0, "includeAll": false, @@ -1436,7 +1436,7 @@ "$__all" ] }, - "datasource": "mysql", + "datasource": {"type": "mysql", "uid": "devlake-mysql-api"}, "definition": "select concat(users.name, '--', users.id) from users left join team_users on users.id = team_users.user_id where team_users.team_id in ($team)", "hide": 0, "includeAll": true, diff --git a/grafana/dashboards/postgresql/Asana.json b/grafana/dashboards/postgresql/Asana.json new file mode 100644 index 00000000000..37650ca6d82 --- /dev/null +++ b/grafana/dashboards/postgresql/Asana.json @@ -0,0 +1,1192 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Asana", + "url": "https://devlake.apache.org/docs/Configuration/Asana" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Asana.\n- Data Source Required: Asana", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total number of issues created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "The cumulative distribution of issue lead time. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Issue Lead Time [Issues Resolved in Selected Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'asana%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'asana%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Asana", + "uid": "asana-dashboard-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Bamboo.json b/grafana/dashboards/postgresql/Bamboo.json new file mode 100644 index 00000000000..469b3fdd2ee --- /dev/null +++ b/grafana/dashboards/postgresql/Bamboo.json @@ -0,0 +1,1219 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 58, + "links": [ + { + "targetBlank": true, + "title": "Bamboo", + "url": "https://devlake.apache.org/docs/Plugins/bamboo" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from Bamboo, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate). A build in Bamboo is an execution of a Bamboo Plan.\n- Data Source Required: Bamboo", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of builds executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Total Number of Successful Builds [Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful builds / Number of total builds", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. Mean Build Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 37, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Total Build Result Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful builds / Number of total builds", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 55, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4. Mean Build Duration in Minutes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 52, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, build_count AS \"Build Count\" FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Total Number of Successful Builds [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Success Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Build Success Rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 50, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", \"result\", id) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Build Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Build Success Rate [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 54, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_build_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of Successful and Failed Builds [Each Month]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Duration(minutes)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_minutes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%bamboo%' AND ('${plan_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${plan_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(mean_duration_sec AS NUMERIC) / NULLIF(60, 0) AS mean_duration_minutes FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.1 Mean Build Duration in Minutes [Each Month]", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 60, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'bamboo%'", + "hide": 0, + "includeAll": true, + "label": "Plan Name", + "multi": true, + "name": "plan_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'bamboo%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Bamboo", + "uid": "a90e58d9-7acc-4858-aa77-f606d11a7d4a-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/DORA.json b/grafana/dashboards/postgresql/DORA.json new file mode 100644 index 00000000000..d1aaa0dc2e9 --- /dev/null +++ b/grafana/dashboards/postgresql/DORA.json @@ -0,0 +1,1517 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 60, + "links": [], + "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See [how to config](https://devlake.apache.org/docs/DORA) this dashboard\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- You can validate/debug this dashboard with the [DORA validation dashboard](/grafana/d/KGkUnV-Vz/dora-dashboard-validation)\n- DORA benchmarks vary in different years. You can switch the benchmarks to change them.\n- In DORA's official report in 2023, metric 'failed deployment recovery time' has replaced 'MTTR'.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "low" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "medium" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "high" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "elite" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": false, + "title": "", + "url": "/d/${__data.fields[\"metric_hidden\"]}?${project:queryparam}&from=${__from}&to=${__to}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Optimized DORA Metrics Query for PostgreSQL/Grafana */\r\n\r\n/* Metric 1: Deployment Frequency */\r\nWITH last_few_calendar_months AS (\r\n /* Generate calendar days efficiently using generate_series instead of cartesian product */\r\n SELECT date::DATE AS day\r\n FROM generate_series(\r\n $__timeFrom()::timestamp,\r\n ($__timeTo()::timestamp - interval '1 day'),\r\n interval '1 day'\r\n ) AS date\r\n),\r\n\r\n_production_deployment_days AS (\r\n /* Get deployment days with single pass aggregation */\r\n SELECT\r\n cdc.cicd_deployment_id AS deployment_id,\r\n MAX(cdc.finished_date::DATE) AS day\r\n FROM cicd_deployment_commits cdc\r\n INNER JOIN project_mapping pm ON\r\n cdc.cicd_scope_id = pm.row_id\r\n AND pm.\"table\" = 'cicd_scopes'\r\n WHERE\r\n cdc.result = 'SUCCESS'\r\n AND cdc.environment = 'PRODUCTION'\r\n AND cdc.finished_date BETWEEN $__timeFrom() AND $__timeTo()\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name = ANY(ARRAY[${project:singlequote}]::text[])\r\n )\r\n GROUP BY cdc.cicd_deployment_id\r\n),\r\n\r\n_deployment_stats AS (\r\n /* Consolidate weekly, monthly, and six-monthly stats in single pass */\r\n SELECT\r\n DATE_TRUNC('week', cal.day)::DATE AS week,\r\n DATE_TRUNC('month', cal.day)::DATE AS month,\r\n cal.day,\r\n dep.day AS deployment_day\r\n FROM last_few_calendar_months cal\r\n LEFT JOIN _production_deployment_days dep ON dep.day = cal.day\r\n),\r\n\r\n_weekly_summary AS (\r\n /* Aggregate weekly deployment days */\r\n SELECT\r\n week,\r\n MAX(CASE WHEN deployment_day IS NOT NULL THEN 1 ELSE 0 END) AS weeks_deployed,\r\n COUNT(DISTINCT deployment_day) AS days_deployed\r\n FROM _deployment_stats\r\n GROUP BY week\r\n),\r\n\r\n_monthly_summary AS (\r\n /* Aggregate monthly deployment days */\r\n SELECT\r\n month,\r\n MAX(CASE WHEN deployment_day IS NOT NULL THEN 1 ELSE NULL END) AS months_deployed,\r\n COUNT(DISTINCT deployment_day) AS days_deployed\r\n FROM _deployment_stats\r\n GROUP BY month\r\n),\r\n\r\n_six_months_rolling AS (\r\n /* Calculate rolling 6-month windows */\r\n SELECT\r\n month,\r\n SUM(days_deployed) OVER (\r\n ORDER BY month\r\n ROWS BETWEEN 5 PRECEDING AND CURRENT ROW\r\n ) AS days_deployed_per_six_months,\r\n COUNT(months_deployed) OVER (\r\n ORDER BY month\r\n ROWS BETWEEN 5 PRECEDING AND CURRENT ROW\r\n ) AS months_deployed_count,\r\n ROW_NUMBER() OVER (\r\n PARTITION BY (EXTRACT(YEAR FROM month)::INTEGER * 12 + EXTRACT(MONTH FROM month)::INTEGER) / 6\r\n ORDER BY month DESC\r\n ) AS rn\r\n FROM _monthly_summary\r\n),\r\n\r\n_six_months_summary AS (\r\n /* Filter to one record per 6-month period */\r\n SELECT\r\n month,\r\n days_deployed_per_six_months,\r\n months_deployed_count\r\n FROM _six_months_rolling\r\n WHERE rn = 1\r\n),\r\n\r\n_median_deployment_frequency AS (\r\n /* Calculate all medians using percentile_cont for efficiency */\r\n SELECT\r\n (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY days_deployed)\r\n FROM _weekly_summary) AS median_weekly,\r\n (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY days_deployed)\r\n FROM _monthly_summary) AS median_monthly,\r\n (SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY days_deployed_per_six_months)\r\n FROM _six_months_summary) AS median_six_monthly,\r\n (SELECT MIN(months_deployed_count)\r\n FROM _six_months_summary\r\n WHERE months_deployed_count > 0) AS is_collected\r\n),\r\n\r\n_metric_deployment_frequency AS (\r\n SELECT\r\n 'Deployment frequency' AS metric,\r\n CASE '$dora_report'\r\n WHEN '2023' THEN\r\n CASE\r\n WHEN median_weekly >= 5 THEN 'On-demand(elite)'\r\n WHEN median_weekly >= 1 THEN 'Between once per day and once per week(high)'\r\n WHEN median_monthly >= 1 THEN 'Between once per week and once per month(medium)'\r\n WHEN median_monthly < 1 AND is_collected IS NOT NULL THEN 'Fewer than once per month(low)'\r\n ELSE 'N/A. Please check if you have collected deployments.'\r\n END\r\n WHEN '2021' THEN\r\n CASE\r\n WHEN median_weekly >= 5 THEN 'On-demand(elite)'\r\n WHEN median_monthly >= 1 THEN 'Between once per day and once per month(high)'\r\n WHEN median_six_monthly >= 1 THEN 'Between once per month and once every 6 months(medium)'\r\n WHEN median_six_monthly < 1 AND is_collected IS NOT NULL THEN 'Fewer than once per six months(low)'\r\n ELSE 'N/A. Please check if you have collected deployments.'\r\n END\r\n ELSE 'Invalid dora report'\r\n END AS value\r\n FROM _median_deployment_frequency\r\n),\r\n\r\n/* Metric 2: Lead Time for Changes - Direct from PRs (Webhook-friendly) */\r\n_pr_cycle_times AS (\r\n /* Calculate lead time directly from merged PRs without deployment dependency */\r\n SELECT ppm.pr_cycle_time\r\n FROM pull_requests pr\r\n INNER JOIN project_pr_metrics ppm ON ppm.id = pr.id\r\n INNER JOIN project_mapping pm ON\r\n pr.base_repo_id = pm.row_id\r\n AND pm.\"table\" = 'repos'\r\n WHERE\r\n pr.merged_date IS NOT NULL\r\n AND ppm.pr_cycle_time IS NOT NULL\r\n AND $__timeFilter(pr.merged_date) -- Filter by PR merge date instead of deployment\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name = ANY(ARRAY[${project:singlequote}]::text[])\r\n )\r\n),\r\n\r\n_metric_change_lead_time AS (\r\n SELECT\r\n 'Lead time for changes' AS metric,\r\n CASE '$dora_report'\r\n WHEN '2023' THEN\r\n CASE\r\n WHEN median_lt IS NULL THEN 'N/A. Please check if you have collected pull_requests.'\r\n WHEN median_lt < 24 * 60 THEN 'Less than one day(elite)'\r\n WHEN median_lt < 7 * 24 * 60 THEN 'Between one day and one week(high)'\r\n WHEN median_lt < 30 * 24 * 60 THEN 'Between one week and one month(medium)'\r\n ELSE 'More than one month(low)'\r\n END\r\n WHEN '2021' THEN\r\n CASE\r\n WHEN median_lt IS NULL THEN 'N/A. Please check if you have collected pull_requests.'\r\n WHEN median_lt < 60 THEN 'Less than one hour(elite)'\r\n WHEN median_lt < 7 * 24 * 60 THEN 'Less than one week(high)'\r\n WHEN median_lt < 180 * 24 * 60 THEN 'Between one week and six months(medium)'\r\n ELSE 'More than six months(low)'\r\n END\r\n ELSE 'Invalid dora report'\r\n END AS value\r\n FROM (\r\n SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY pr_cycle_time) AS median_lt\r\n FROM _pr_cycle_times\r\n ) AS medians\r\n),\r\n\r\n/* Metric 3: Change Failure Rate */\r\n_deployments AS (\r\n SELECT\r\n cdc.cicd_deployment_id AS deployment_id,\r\n MAX(cdc.finished_date) AS deployment_finished_date\r\n FROM cicd_deployment_commits cdc\r\n INNER JOIN project_mapping pm ON\r\n cdc.cicd_scope_id = pm.row_id\r\n AND pm.\"table\" = 'cicd_scopes'\r\n WHERE\r\n cdc.result = 'SUCCESS'\r\n AND cdc.environment = 'PRODUCTION'\r\n AND cdc.finished_date BETWEEN $__timeFrom() AND $__timeTo()\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name = ANY(ARRAY[${project:singlequote}]::text[])\r\n )\r\n GROUP BY cdc.cicd_deployment_id\r\n),\r\n\r\n_change_failure_rate AS (\r\n SELECT\r\n COALESCE(\r\n COUNT(DISTINCT CASE WHEN i.id IS NOT NULL THEN d.deployment_id END)::NUMERIC\r\n / NULLIF(COUNT(DISTINCT d.deployment_id), 0),\r\n 0\r\n ) AS cfr,\r\n COUNT(DISTINCT d.deployment_id) AS deployment_count,\r\n COUNT(DISTINCT i.id) AS incident_count\r\n FROM _deployments d\r\n LEFT JOIN project_incident_deployment_relationships pim ON d.deployment_id = pim.deployment_id\r\n LEFT JOIN incidents i ON pim.id = i.id\r\n),\r\n\r\n_metric_cfr AS (\r\n SELECT\r\n 'Change failure rate' AS metric,\r\n CASE\r\n WHEN deployment_count = 0 AND incident_count = 0 THEN\r\n 'N/A. Please check if you have collected deployments/incidents.'\r\n WHEN incident_count = 0 THEN\r\n 'N/A. Please check if you have collected incidents.'\r\n WHEN deployment_count = 0 THEN\r\n 'N/A. Please check if you have collected deployments.'\r\n WHEN '$dora_report' = '2023' THEN\r\n CASE\r\n WHEN cfr <= 0.05 THEN '0-5%(elite)'\r\n WHEN cfr <= 0.10 THEN '5%-10%(high)'\r\n WHEN cfr <= 0.15 THEN '10%-15%(medium)'\r\n ELSE '> 15%(low)'\r\n END\r\n WHEN '$dora_report' = '2021' THEN\r\n CASE\r\n WHEN cfr <= 0.15 THEN '0-15%(elite)'\r\n WHEN cfr <= 0.20 THEN '16%-20%(high)'\r\n WHEN cfr <= 0.30 THEN '21%-30%(medium)'\r\n ELSE '> 30%(low)'\r\n END\r\n ELSE 'Invalid dora report'\r\n END AS value\r\n FROM _change_failure_rate\r\n),\r\n\r\n/* Metric 4: Recovery Time (2023) / MTTR (2021) */\r\n_recovery_times_2023 AS (\r\n SELECT\r\n EXTRACT(EPOCH FROM (i.resolution_date - d.deployment_finished_date)) / 60 AS recovery_minutes\r\n FROM incidents i\r\n INNER JOIN project_incident_deployment_relationships pim ON i.id = pim.id\r\n INNER JOIN _deployments d ON pim.deployment_id = d.deployment_id\r\n WHERE\r\n i.resolution_date IS NOT NULL\r\n AND $__timeFilter(i.resolution_date)\r\n),\r\n\r\n_metric_recovery_time_2023 AS (\r\n SELECT\r\n 'Failed deployment recovery time' AS metric,\r\n CASE\r\n WHEN median_rt < 60 THEN 'Less than one hour(elite)'\r\n WHEN median_rt < 24 * 60 THEN 'Less than one day(high)'\r\n WHEN median_rt < 7 * 24 * 60 THEN 'Between one day and one week(medium)'\r\n WHEN median_rt >= 7 * 24 * 60 THEN 'More than one week(low)'\r\n ELSE 'N/A. Please check if you have collected deployments or incidents.'\r\n END AS value\r\n FROM (\r\n SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY recovery_minutes) AS median_rt\r\n FROM _recovery_times_2023\r\n ) AS medians\r\n WHERE '$dora_report' = '2023'\r\n),\r\n\r\n_mttr_2021 AS (\r\n SELECT\r\n 'Time to restore service' AS metric,\r\n CASE\r\n WHEN median_mttr < 60 THEN 'Less than one hour(elite)'\r\n WHEN median_mttr < 24 * 60 THEN 'Less than one day(high)'\r\n WHEN median_mttr < 7 * 24 * 60 THEN 'Between one day and one week(medium)'\r\n WHEN median_mttr >= 7 * 24 * 60 THEN 'More than one week(low)'\r\n ELSE 'N/A. Please check if you have collected incidents.'\r\n END AS value\r\n FROM (\r\n SELECT PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY lead_time_minutes) AS median_mttr\r\n FROM incidents i\r\n INNER JOIN project_mapping pm ON\r\n i.scope_id = pm.row_id\r\n AND pm.\"table\" = i.\"table\"\r\n WHERE\r\n i.lead_time_minutes IS NOT NULL\r\n AND $__timeFilter(i.resolution_date)\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name = ANY(ARRAY[${project:singlequote}]::text[])\r\n )\r\n ) AS medians\r\n WHERE '$dora_report' = '2021'\r\n),\r\n\r\n_all_metrics AS (\r\n SELECT metric, value FROM _metric_deployment_frequency\r\n UNION ALL\r\n SELECT metric, value FROM _metric_change_lead_time\r\n UNION ALL\r\n SELECT metric, value FROM _metric_cfr\r\n UNION ALL\r\n SELECT metric, value FROM _metric_recovery_time_2023\r\n UNION ALL\r\n SELECT metric, value FROM _mttr_2021\r\n)\r\n\r\n/* Final output with benchmark comparison */\r\nSELECT\r\n db.metric,\r\n REPLACE(db.metric, ' ', '-') AS metric_hidden,\r\n CASE WHEN db.low = m.value THEN db.low END AS low,\r\n CASE WHEN db.medium = m.value THEN db.medium END AS medium,\r\n CASE WHEN db.high = m.value THEN db.high END AS high,\r\n CASE WHEN db.elite = m.value THEN db.elite END AS elite\r\nFROM dora_benchmarks db\r\nINNER JOIN _all_metrics m ON db.metric = m.metric\r\nWHERE db.dora_report = '$dora_report'\r\nORDER BY db.id;\r\n\r\n/* Recommended indexes for optimal performance:\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cicd_deployment_commits_lookup\r\n ON cicd_deployment_commits(cicd_scope_id, result, environment, finished_date)\r\n INCLUDE (cicd_deployment_id);\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_project_mapping_scope\r\n ON project_mapping(row_id, \"table\", project_name);\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pull_requests_merge\r\n ON pull_requests(base_repo_id, merged_date)\r\n WHERE merged_date IS NOT NULL;\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_project_pr_metrics_deployment\r\n ON project_pr_metrics(deployment_commit_id, pr_cycle_time)\r\n WHERE pr_cycle_time IS NOT NULL;\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_incidents_resolution\r\n ON incidents(scope_id, resolution_date, lead_time_minutes)\r\n WHERE resolution_date IS NOT NULL;\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_project_incident_deployment\r\n ON project_incident_deployment_relationships(deployment_id, id);\r\n*/\r\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Overall DORA Metrics", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "How often an organization deploys code to production or release it to end users\n\n# 2023 Benchmarks\n- **Elite**: >= 7 deployment days per week\n- **High**: >= 1 deployment day per week\n- **Medium**: >= 1 deployment day per month\n- **Low**: < 1 deployment day per month\n# 2021 Benchmarks\n- **Elite**: >= 7 deployment days per week\n- **High**: >= 1 deployment day per month\n- **Medium**: >= 1 deployment day per six months\n- **Low**: < 1 deployment day per six months", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 11, + "links": [ + { + "targetBlank": false, + "title": "link", + "url": "/d/Deployment-frequency/dora-drill-down-deployment-frequency?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Deployment Frequency$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Optimized Deployment Frequency Query for PostgreSQL/Grafana */\r\n\r\nWITH last_few_calendar_months AS (\r\n /* Generate calendar days efficiently using generate_series */\r\n SELECT date::DATE AS day\r\n FROM generate_series(\r\n $__timeFrom()::timestamp,\r\n ($__timeTo()::timestamp - INTERVAL '1 day'),\r\n INTERVAL '1 day'\r\n ) AS date\r\n),\r\n\r\n_production_deployment_days AS (\r\n /* Get unique deployment days - GitLab/BitBucket may generate multiple deployments for one pipeline */\r\n SELECT\r\n cdc.cicd_deployment_id AS deployment_id,\r\n MAX(cdc.finished_date::DATE) AS day\r\n FROM cicd_deployment_commits cdc\r\n INNER JOIN project_mapping pm ON\r\n cdc.cicd_scope_id = pm.row_id\r\n AND pm.\"table\" = 'cicd_scopes'\r\n WHERE\r\n cdc.result = 'SUCCESS'\r\n AND cdc.environment = 'PRODUCTION'\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name = ANY(ARRAY[${project:singlequote}]::text[])\r\n )\r\n GROUP BY cdc.cicd_deployment_id\r\n),\r\n\r\n_days_weekly_deploy AS (\r\n /* Calculate deployment days per week */\r\n SELECT\r\n DATE_TRUNC('week', cal.day)::DATE AS week,\r\n MAX(CASE WHEN dep.day IS NOT NULL THEN 1 ELSE NULL END) AS weeks_deployed,\r\n COUNT(DISTINCT dep.day) AS days_deployed\r\n FROM last_few_calendar_months cal\r\n LEFT JOIN _production_deployment_days dep ON dep.day = cal.day\r\n GROUP BY week\r\n),\r\n\r\n_days_monthly_deploy AS (\r\n /* Calculate deployment days per month */\r\n SELECT\r\n DATE_TRUNC('month', cal.day)::DATE AS month,\r\n MAX(CASE WHEN dep.day IS NOT NULL THEN 1 ELSE NULL END) AS months_deployed,\r\n COUNT(DISTINCT dep.day) AS days_deployed\r\n FROM last_few_calendar_months cal\r\n LEFT JOIN _production_deployment_days dep ON dep.day = cal.day\r\n GROUP BY month\r\n),\r\n\r\n_days_six_months_rolling AS (\r\n /* Calculate rolling 6-month deployment windows */\r\n SELECT\r\n month,\r\n SUM(days_deployed) OVER (\r\n ORDER BY month\r\n ROWS BETWEEN 5 PRECEDING AND CURRENT ROW\r\n ) AS days_deployed_per_six_months,\r\n COUNT(months_deployed) OVER (\r\n ORDER BY month\r\n ROWS BETWEEN 5 PRECEDING AND CURRENT ROW\r\n ) AS months_deployed_count,\r\n ROW_NUMBER() OVER (\r\n PARTITION BY (EXTRACT(YEAR FROM month)::INTEGER * 12 + EXTRACT(MONTH FROM month)::INTEGER) / 6\r\n ORDER BY month DESC\r\n ) AS rn\r\n FROM _days_monthly_deploy\r\n),\r\n\r\n_days_per_six_months_deploy_by_filter AS (\r\n /* Filter to one record per 6-month period */\r\n SELECT\r\n month,\r\n days_deployed_per_six_months,\r\n months_deployed_count\r\n FROM _days_six_months_rolling\r\n WHERE rn = 1\r\n),\r\n\r\n_medians AS (\r\n /* Calculate all medians in one pass using percentile_cont */\r\n SELECT\r\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY w.days_deployed) AS median_weekly,\r\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY m.days_deployed) AS median_monthly,\r\n PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY s.days_deployed_per_six_months) AS median_six_monthly,\r\n MIN(s.months_deployed_count) AS is_collected\r\n FROM _days_weekly_deploy w\r\n CROSS JOIN _days_monthly_deploy m\r\n CROSS JOIN _days_per_six_months_deploy_by_filter s\r\n)\r\n\r\n/* Final output with classification */\r\nSELECT\r\n CASE '$dora_report'\r\n WHEN '2023' THEN\r\n CASE\r\n WHEN median_weekly >= 5 THEN\r\n median_weekly || ' deployment days per week(elite)'\r\n WHEN median_weekly >= 1 THEN\r\n median_weekly || ' deployment days per week(high)'\r\n WHEN median_monthly >= 1 THEN\r\n median_monthly || ' deployment days per month(medium)'\r\n WHEN median_monthly < 1 AND is_collected IS NOT NULL THEN\r\n median_monthly || ' deployment days per month(low)'\r\n ELSE\r\n 'N/A. Please check if you have collected deployments.'\r\n END\r\n WHEN '2021' THEN\r\n CASE\r\n WHEN median_weekly >= 5 THEN\r\n median_weekly || ' deployment days per week(elite)'\r\n WHEN median_monthly >= 1 THEN\r\n median_monthly || ' deployment days per month(high)'\r\n WHEN median_six_monthly >= 1 THEN\r\n median_six_monthly || ' deployment days per six months(medium)'\r\n WHEN median_six_monthly < 1 AND is_collected IS NOT NULL THEN\r\n median_six_monthly || ' deployment days per six months(low)'\r\n ELSE\r\n 'N/A. Please check if you have collected deployments.'\r\n END\r\n ELSE\r\n 'Invalid dora report'\r\n END AS \"Deployment Frequency\"\r\nFROM _medians;\r\n\r\n/* Performance tip: Consider creating these indexes:\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_cicd_deployment_commits_prod\r\n ON cicd_deployment_commits(cicd_scope_id, result, environment, finished_date)\r\n WHERE result = 'SUCCESS' AND environment = 'PRODUCTION';\r\n\r\nCREATE INDEX CONCURRENTLY IF NOT EXISTS idx_project_mapping_cicd_scopes\r\n ON project_mapping(row_id, project_name)\r\n WHERE \"table\" = 'cicd_scopes';\r\n*/\r\n", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Deployment Frequency", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "The median amount of time for a code change to be deployed into production.\n\n\n# 2023 Benchmarks\n- **Elite**: Less than one day\n- **High**: Between one day and one week\n- **Medium**: Between one week and one month\n- **Low**: More than one month\n# 2021 Benchmarks\n- **Elite**: Less than one hour\n- **High**: Less than one week\n- **Medium**: Between one week and six months\n- **Low**: More than six months\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 12, + "links": [ + { + "title": "link", + "url": "/d/Lead-time-for-changes/dora-drill-down-lead-time-for-changes?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Optimized Lead Time for Changes - PostgreSQL + Grafana */\r\n/* Webhook-friendly version with formatted DORA output */\r\n\r\nWITH _pr_cycle_times AS (\r\n\r\n SELECT DISTINCT\r\n pr.id,\r\n ppm.pr_cycle_time\r\n FROM pull_requests pr\r\n\r\n INNER JOIN project_pr_metrics ppm\r\n ON ppm.id = pr.id\r\n\r\n INNER JOIN project_mapping pm\r\n ON pr.base_repo_id = pm.row_id\r\n AND pm.\"table\" = 'repos'\r\n\r\n /* Optional deployment validation */\r\n LEFT JOIN cicd_deployment_commits cdc\r\n ON ppm.deployment_commit_id = cdc.id\r\n\r\n WHERE\r\n pr.merged_date IS NOT NULL\r\n AND ppm.pr_cycle_time IS NOT NULL\r\n\r\n /* Better for webhook scenarios */\r\n AND $__timeFilter(pr.merged_date)\r\n\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name::text = ANY(\r\n ARRAY[${project:singlequote}]::text[]\r\n )\r\n )\r\n\r\n),\r\n\r\n_median_change_lead_time_ranks AS (\r\n\r\n SELECT\r\n *,\r\n PERCENT_RANK() OVER (\r\n ORDER BY pr_cycle_time NULLS FIRST\r\n ) AS ranks\r\n FROM _pr_cycle_times\r\n\r\n),\r\n\r\n_median_change_lead_time AS (\r\n\r\n SELECT\r\n MAX(pr_cycle_time) AS median_change_lead_time\r\n FROM _median_change_lead_time_ranks\r\n WHERE ranks <= 0.5\r\n\r\n)\r\n\r\nSELECT\r\n\r\n CASE\r\n\r\n WHEN ('${dora_report}') = '2023' THEN\r\n\r\n CASE\r\n\r\n WHEN median_change_lead_time < 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (elite)'\r\n\r\n WHEN median_change_lead_time < 7 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (high)'\r\n\r\n WHEN median_change_lead_time < 30 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (medium)'\r\n\r\n WHEN median_change_lead_time >= 30 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (low)'\r\n\r\n ELSE\r\n 'N/A. Please check if you have collected pull_requests.'\r\n\r\n END\r\n\r\n WHEN ('${dora_report}') = '2021' THEN\r\n\r\n CASE\r\n\r\n WHEN median_change_lead_time < 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (elite)'\r\n\r\n WHEN median_change_lead_time < 7 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (high)'\r\n\r\n WHEN median_change_lead_time < 180 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (medium)'\r\n\r\n WHEN median_change_lead_time >= 180 * 24 * 60 THEN\r\n ROUND(\r\n CAST(\r\n CAST(median_change_lead_time AS NUMERIC) / 60\r\n AS DECIMAL\r\n ),\r\n 1\r\n ) || ' hours (low)'\r\n\r\n ELSE\r\n 'N/A. Please check if you have collected pull_requests.'\r\n\r\n END\r\n\r\n ELSE\r\n 'Invalid dora report'\r\n\r\n END AS median_change_lead_time\r\n\r\nFROM _median_change_lead_time;", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Median Lead Time for Changes In Hours", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "The percentage of changes that were made to a code that then resulted in incidents, rollbacks, or any type of production failure.\n\n# 2023 Benchmarks\n- **Elite**: (0, 5%]\n- **High**: (5%, 10%]\n- **Medium**: (10%, 15%]\n- **Low**: (15%, 100%]\n# 2021 Benchmarks\n- **Elite**: (0, 15%]\n- **High**: (16%, 20%]\n- **Medium**: (21%, 30%]\n- **Low**: (30%, 100%]\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 14, + "links": [ + { + "title": "link", + "url": "/d/Change-failure-rate/dora-drill-down-change-failure-rate?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^change_failure_rate$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate AS (SELECT CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments), _is_collected_data AS (SELECT CASE WHEN COUNT(i.id) = 0 AND COUNT(cdc.id) = 0 THEN 'No All' WHEN COUNT(i.id) = 0 THEN 'No Incidents' WHEN COUNT(cdc.id) = 0 THEN 'No Deployments' END AS is_collected FROM (SELECT 1) AS dummy LEFT JOIN incidents AS i ON 1 = 1 LEFT JOIN cicd_deployment_commits AS cdc ON 1 = 1) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.05 THEN ROUND(change_failure_rate * 100, 1) || '%(elite)' WHEN change_failure_rate <= 0.10 THEN ROUND(change_failure_rate * 100, 1) || '%(high)' WHEN change_failure_rate <= 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(medium)' WHEN change_failure_rate > 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(elite)' WHEN change_failure_rate <= 0.20 THEN ROUND(change_failure_rate * 100, 1) || '%(high)' WHEN change_failure_rate <= 0.30 THEN ROUND(change_failure_rate * 100, 1) || '%(medium)' WHEN change_failure_rate > 0.30 THEN ROUND(change_failure_rate * 100, 1) || '%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END ELSE 'No data' END AS change_failure_rate FROM _change_failure_rate, \"_is_collected_data\"", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Change Failure Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "# MTTR benchmarks:\ndevlake.apache.org/docs/Metrics/MTTR#how-is-it-calculated\n# FDRT benchmarks:\ndevlake.apache.org/docs/Metrics/FailedDeploymentRecoveryTime#how-is-it-calculated\n", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 17, + "links": [ + { + "title": "Failed Deployment Recovery Time", + "url": "/d/Failed-deployment-recovery-time/dora-details-failed-deployment-recovery-time?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + }, + { + "title": "Median Time to Restore Service", + "url": "/d/Time-to-restore-service/dora-details-median-time-to-restore-service?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5), _is_collected_data AS (SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM _deployments) AND NOT EXISTS(SELECT 1 FROM incidents) THEN 'No deployments and incidents' WHEN NOT EXISTS(SELECT 1 FROM _deployments) THEN 'No Deployments' WHEN NOT EXISTS(SELECT 1 FROM incidents) THEN 'No Incidents' ELSE 'No incidents are mapped to deployments' END AS is_collected FROM _deployments AS d, _incidents_for_deployments AS i), _metric_recovery_time_2023_report AS (SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No deployments and incidents' THEN 'N/A. Please check if you have collected deployments and incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN median_recovery_time < 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_recovery_time < 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_recovery_time < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_recovery_time >= 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)' ELSE 'No data' END END AS median_recovery_time FROM _median_recovery_time, _is_collected_data), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.resolution_date)), _median_mttr_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _median_mttr AS (SELECT MAX(lead_time_minutes) AS median_time_to_resolve FROM _median_mttr_ranks WHERE ranks <= 0.5), _metric_mttr_2021_report AS (SELECT CASE WHEN ('$dora_report') = '2021' THEN CASE WHEN median_time_to_resolve < 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_time_to_resolve < 24 * 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_time_to_resolve < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_time_to_resolve >= 7 * 24 * 60 THEN (ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)')::TEXT ELSE 'N/A. Please check if you have collected incidents.' END END AS median_time_to_resolve FROM _median_mttr) SELECT median_recovery_time AS median_time_in_hour FROM _metric_recovery_time_2023_report WHERE ('$dora_report') = '2023' UNION SELECT median_time_to_resolve AS median_time_to_resolve FROM _metric_mttr_2021_report WHERE ('$dora_report') = '2021'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "${title_value} In Hours", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 2, + "links": [ + { + "title": "link", + "url": "/d/Deployment-frequency/dora-drill-down-deployment-frequency?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Number of deployments per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, COUNT(cicd_deployment_id) AS deployment_count FROM (SELECT cdc.cicd_deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS _production_deployments GROUP BY 1) SELECT cm.month, CASE WHEN d.deployment_count IS NULL THEN 0 ELSE d.deployment_count END AS \"Deployment Count\" FROM calendar_months AS cm LEFT JOIN _deployments AS d ON cm.month = d.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Monthly deployments", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 6, + "links": [ + { + "title": "link", + "url": "/d/Lead-time-for-changes/dora-drill-down-lead-time-for-changes?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* Metric 2: Median Change Lead Time per Month */\r\n/* PostgreSQL + Grafana + Webhook-friendly */\r\n\r\nWITH _pr_stats AS (\r\n\r\n /* Get PR cycle time grouped by merged month */\r\n SELECT DISTINCT\r\n pr.id,\r\n\r\n TO_CHAR(\r\n DATE_TRUNC('month', pr.merged_date),\r\n 'YY/MM'\r\n ) AS month,\r\n\r\n ppm.pr_cycle_time\r\n\r\n FROM pull_requests pr\r\n\r\n INNER JOIN project_pr_metrics ppm\r\n ON ppm.id = pr.id\r\n\r\n INNER JOIN project_mapping pm\r\n ON pr.base_repo_id = pm.row_id\r\n AND pm.\"table\" = 'repos'\r\n\r\n /* Optional deployment linkage */\r\n LEFT JOIN cicd_deployment_commits cdc\r\n ON ppm.deployment_commit_id = cdc.id\r\n\r\n WHERE\r\n pr.merged_date IS NOT NULL\r\n AND ppm.pr_cycle_time IS NOT NULL\r\n\r\n /* Better compatibility with webhook deployments */\r\n AND $__timeFilter(pr.merged_date)\r\n\r\n AND (\r\n '${project:csv}' = ''\r\n OR pm.project_name::text = ANY(\r\n ARRAY[${project:singlequote}]::text[]\r\n )\r\n )\r\n\r\n),\r\n\r\n_find_median_clt_each_month_ranks AS (\r\n\r\n SELECT\r\n *,\r\n PERCENT_RANK() OVER (\r\n PARTITION BY month\r\n ORDER BY pr_cycle_time NULLS FIRST\r\n ) AS ranks\r\n FROM _pr_stats\r\n\r\n),\r\n\r\n_clt AS (\r\n\r\n SELECT\r\n month,\r\n MAX(pr_cycle_time) AS median_change_lead_time\r\n FROM _find_median_clt_each_month_ranks\r\n WHERE ranks <= 0.5\r\n GROUP BY month\r\n\r\n)\r\n\r\nSELECT\r\n\r\n cm.month,\r\n\r\n CASE\r\n WHEN _clt.median_change_lead_time IS NULL THEN\r\n 0\r\n\r\n ELSE\r\n ROUND(\r\n CAST(\r\n _clt.median_change_lead_time AS NUMERIC\r\n ) / 60,\r\n 1\r\n )\r\n END AS \"Median Change Lead Time In Hours\"\r\n\r\nFROM calendar_months cm\r\n\r\nLEFT JOIN _clt\r\n ON cm.month = _clt.month\r\n\r\nWHERE\r\n cm.month_timestamp BETWEEN\r\n DATE_TRUNC(\r\n 'month',\r\n CAST($__timeFrom() AS TIMESTAMP)\r\n )\r\n AND\r\n DATE_TRUNC(\r\n 'month',\r\n CAST($__timeTo() AS TIMESTAMP)\r\n )\r\n\r\nORDER BY cm.month_timestamp;", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Median Lead Time for Changes", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "change_failure_rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 5, + "links": [ + { + "title": "link", + "url": "/d/Change-failure-rate/dora-drill-down-change-failure-rate?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate_for_each_month AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments GROUP BY 1) SELECT cm.month, cfr.change_failure_rate AS \"Change Failure Rate\" FROM calendar_months AS cm LEFT JOIN _change_failure_rate_for_each_month AS cfr ON cm.month = cfr.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Change Failure Rate", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "median_time_to_resolve_in_hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 9, + "links": [ + { + "title": "Failed Deployment Recovery Time", + "url": "/d/Failed-deployment-recovery-time/dora-details-failed-deployment-recovery-time?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + }, + { + "title": "Median Time To Restore Service", + "url": "/d/Time-to-restore-service/dora-details-median-time-to-restore-service?orgId=1&${project:queryparam}&from=${__from}&to=${__to}" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "hideZeros": false, + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY deployment_finished_month ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT deployment_finished_month, MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5 GROUP BY deployment_finished_month), _metric_recovery_time_2023_report AS (SELECT cm.month, CASE WHEN m.median_recovery_time IS NULL THEN 0 ELSE CAST(m.median_recovery_time AS NUMERIC) / NULLIF(60, 0) END AS median_recovery_time_in_hour FROM calendar_months AS cm LEFT JOIN _median_recovery_time AS m ON cm.month = m.deployment_finished_month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: median time to restore service - MTTR */ AS (/* get the number of incidents created each month */ SELECT DISTINCT i.id, TO_CHAR(CAST(i.resolution_date AS TIMESTAMP), 'YY/MM') AS month, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT i.lead_time_minutes IS NULL), _find_median_mttr_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _mttr AS (SELECT month, MAX(lead_time_minutes) AS median_time_to_resolve FROM _find_median_mttr_each_month_ranks WHERE ranks <= 0.5 GROUP BY month), _metric_mttr_2021_report AS (SELECT cm.month, CASE WHEN m.median_time_to_resolve IS NULL THEN 0 ELSE CAST(m.median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) END AS median_time_to_resolve_in_hour FROM calendar_months AS cm LEFT JOIN _mttr AS m ON cm.month = m.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)) SELECT cm.month, CASE WHEN '${dora_report}' = '2023' THEN mrt.median_recovery_time_in_hour WHEN '${dora_report}' = '2021' THEN mm.median_time_to_resolve_in_hour END AS \"${title_value} In Hours\" FROM calendar_months AS cm LEFT JOIN _metric_recovery_time_2023_report AS mrt ON cm.month = mrt.month LEFT JOIN _metric_mttr_2021_report AS mm ON cm.month = mm.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "${title_value}", + "type": "barchart" + } + ], + "preload": false, + "refresh": "", + "schemaVersion": 41, + "tags": [ + "Engineering Leads Dashboard", + "Highlights", + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "text": "All", + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "includeAll": false, + "label": "DORA Report", + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "type": "query" + }, + { + "current": { + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "DORA", + "uid": "qNo8_0M4z-pgr6", + "version": 2 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Gitlab.json b/grafana/dashboards/postgresql/Gitlab.json new file mode 100644 index 00000000000..af78160fc19 --- /dev/null +++ b/grafana/dashboards/postgresql/Gitlab.json @@ -0,0 +1,2327 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 101, + "links": [ + { + "targetBlank": true, + "title": "GitLab", + "url": "https://devlake.apache.org/docs/Plugins/gitlab" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from GitLab.\n- Data Source Required: GitLab", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 83, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Contribution (PRs)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 68, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Number of New Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pull Request Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 4 + }, + "id": 77, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT id) AS pr_count FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(created_date) /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, pr_count AS \"Pull Request Count\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.2 Number of New Pull Requests", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Merged PR Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 54, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT author_name, COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'MERGED' GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.3 Top 20 Contributors By Merged PRs", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 85, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. How Merge Requests(MRs) are handled?", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 66, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Ratio of unmerged MRs of All Closed or Merged PRs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "MR: Open" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MR: Closed without merging" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MR: Closed and merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 17 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status = 'OPEN' THEN id ELSE NULL END) AS open, COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS closed, COUNT(DISTINCT CASE WHEN status = 'MERGED' THEN id ELSE NULL END) AS merged FROM pull_requests WHERE $__timeFilter(created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, open AS \"MR: Open\", closed AS \"MR: Closed without merging\", merged AS \"MR: Closed and merged\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.2 Merge Request Status Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 80, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'CLOSED'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.3 Number of MRs Closed without Merging", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ratio", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 23 + }, + "id": 81, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.4 Ratio of unmerged MRs of All Closed or Merged MRs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 72, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.5 Mean Time to Merge of Merge Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "dashed" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 29 + }, + "id": 95, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_merge FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, time_to_merge AS \"Time to Merge\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.6 Mean Time to Merge of MRs", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 110, + "panels": [], + "title": "CI/CD Metrics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of pipeline runs executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 102, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of successful pipeline runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful pipeline runs / Number of total pipeline runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 103, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* Enable the following condition to remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.2 Mean pipeline run success rate %", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 108, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.25, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, \"successful_count\", failed_count FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.3 Number of successful and failed pipeline runs", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 104, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN result = '' THEN 'Unknown' ELSE result END AS result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.4 Pipeline runs result distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 42 + }, + "id": 107, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY \"time\", \"result\", id) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN TO_CHAR(CAST(time AS TIMESTAMP), 'MM/YYYY') ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Pipeline Runs Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.5 Pipeline run success rate %", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful pipeline runs / Number of total pipeline runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 105, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* Enable the following condition to remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.6 Mean pipeline runs duration", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_sec" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 48 + }, + "id": 109, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%gitlab%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, mean_duration_sec FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.7 Mean pipeline run duration", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 99, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'gitlab%'", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'gitlab%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Month", + "value": "DAYOFMONTH" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "GitLab", + "uid": "msSjEq97z-pg", + "version": 27, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Homepage.json b/grafana/dashboards/postgresql/Homepage.json new file mode 100644 index 00000000000..33902882efd --- /dev/null +++ b/grafana/dashboards/postgresql/Homepage.json @@ -0,0 +1,310 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 26, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 32, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Welcome\n\n - Dashboards are grouped by use cases and data sources.\n - All data is queried from here. You can go to the 'Explore' page on the left menu to play around.\n - You can always make your own dashboard based on the domain layer schema, SQL examples in engineering metrics, and dashboard manuals.\n", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 22, + "options": { + "folderId": 0, + "maxItems": 10, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": false, + "showStarred": true, + "tags": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Starred Dashboards", + "type": "dashlist" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 2, + "options": { + "folderId": 0, + "maxItems": 50, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [ + "Highlights" + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Highlights", + "type": "dashlist" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "By Roles", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 41, + "options": { + "maxItems": 10, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [ + "Engineering Leads Dashboard" + ] + }, + "pluginVersion": "13.0.2", + "title": "For Engineering Leads", + "type": "dashlist" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 42, + "options": { + "maxItems": 10, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [ + "OSS Maintainer Dashboard" + ] + }, + "pluginVersion": "13.0.2", + "title": "For OSS Maintainers", + "type": "dashlist" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 43, + "options": { + "maxItems": 10, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [ + "Developer" + ] + }, + "pluginVersion": "13.0.2", + "title": "For Developers", + "type": "dashlist" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "By Data Sources", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 40, + "options": { + "maxItems": 20, + "query": "", + "showHeadings": false, + "showRecentlyViewed": false, + "showSearch": true, + "showStarred": false, + "tags": [ + "Data Source Dashboard" + ] + }, + "pluginVersion": "13.0.2", + "type": "dashlist" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Homepage", + "uid": "lCO8w-pVk-pg", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Jenkins.json b/grafana/dashboards/postgresql/Jenkins.json new file mode 100644 index 00000000000..ccc85722e26 --- /dev/null +++ b/grafana/dashboards/postgresql/Jenkins.json @@ -0,0 +1,1202 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 30, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 58, + "links": [ + { + "targetBlank": true, + "title": "Jenkins", + "url": "https://devlake.apache.org/docs/Plugins/jenkins" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from Jenkins, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate).\n- Data Source Required: Jenkins", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of builds executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Total Number of Successful Builds [Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful builds / Number of total builds", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. Mean Build Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 37, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Total Build Result Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful builds / Number of total builds", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 55, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4. Mean Build Duration in Minutes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 52, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, build_count AS \"Build Count\" FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Total Number of Successful Builds [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Build success rate over time.\n2. The builds being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Success Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Build Success Rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 50, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", \"result\", id) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Build Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Build Success Rate [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 54, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_build_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of Successful and Failed Builds [Each Month]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Duration(minutes)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_minutes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%jenkins%' AND ('${job_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${job_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(mean_duration_sec AS NUMERIC) / NULLIF(60, 0) AS mean_duration_minutes FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.1 Mean Build Duration in Minutes [Each Month]", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 60, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'jenkins%'", + "hide": 0, + "includeAll": true, + "label": "Job Name", + "multi": true, + "name": "job_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'jenkins%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Jenkins", + "uid": "W8AiDFQnk-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Jira.json b/grafana/dashboards/postgresql/Jira.json new file mode 100644 index 00000000000..1b200dbb2f3 --- /dev/null +++ b/grafana/dashboards/postgresql/Jira.json @@ -0,0 +1,1248 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 20, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Jira", + "url": "https://devlake.apache.org/docs/Plugins/jira" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Jira.\n- Data Source Required: Jira", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 15, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'jira%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'jira%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "Jira", + "uid": "F5vqBQl7z-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Opsgenie.json b/grafana/dashboards/postgresql/Opsgenie.json new file mode 100644 index 00000000000..f76531c812b --- /dev/null +++ b/grafana/dashboards/postgresql/Opsgenie.json @@ -0,0 +1,1536 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 12, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Opsgenie", + "url": "https://devlake.apache.org/docs/Plugins/opsgenie" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the incident data from Opsgenie.\n- Data Source Required: Opsgenie", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Incident Resolution Status", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Opsgenie board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.issue_key) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.issue_key) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Resolved Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 246 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 131, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.title AS \"Title\", i.issue_key AS \"Id\", STRING_AGG(b.name, ', ') AS \"Service(s)\", i.description AS \"Description\", i.original_status AS \"Original Status\", i.priority AS \"Priority\", i.created_date AS \"Created Date\", i.updated_date AS \"Updated Date\", ROUND(CAST((CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS DECIMAL), 1) AS \"Lead Time Days\", i.url AS \"URL\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY i.title, i.issue_key, i.description, i.original_status, i.priority, i.created_date, i.updated_date, i.lead_time_minutes, i.url", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of Incidents [Created in Selected Time Range]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 7 + }, + "id": 134, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.issue_key) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'closed' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Closed Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.original_status = 'resolved' THEN i.id ELSE NULL END) AS resolved_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * resolved_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate [Incidents created in the selected time range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Resolution Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.original_status = 'resolved' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS resolved_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, resolved_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate over Time [Incidents Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Mean Time to Resolve (MTTR)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "MTTR [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Incidents' MTTR are less than # [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Incident Age(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_incident_age FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_incident_age FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean MTTR [Incidents Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of MTTR\n2. Each point refers to the percent rank of a distinct duration to resolve incidents.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of MTTR [Incidents Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 31 + }, + "id": 132, + "panels": [], + "title": "3. Incident Responders", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Due to Opsgenie API limitation regarding the relationship between assignee, users, and incident (and lack of 'assign at' key ), the column assignee represents a JSON object containing all assignee from said incident", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "json-view" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "issue_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 504 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width", + "value": 286 + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 133, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT i.title AS \"Incident Name\", '[' || STRING_AGG('{\"name\": \"' || a.assignee_name || '\"' || ', \"type\": \"' || r.type || '\"}', ', ') || ']' AS \"Responders\" FROM issues AS i INNER JOIN issue_assignees AS a ON (REVERSE(SPLIT_PART(REVERSE(i.id), ':', 1)) = REVERSE(SPLIT_PART(REVERSE(a.issue_id), ':', 1))) INNER JOIN _tool_opsgenie_responders AS r ON a.assignee_id = r.id WHERE i.id::text LIKE 'opsgenie%' GROUP BY i.title", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "List of Incident Responder [organized in JSON format]", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'opsgenie%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'opsgenie%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "2021-06-05T21:00:42.390Z", + "to": "2025-06-16T21:00:42.390Z" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Opsgenie", + "uid": "b4556439-f173-4411-93d4-65f261726d24-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Sonarqube.json b/grafana/dashboards/postgresql/Sonarqube.json new file mode 100644 index 00000000000..b9b916f14ca --- /dev/null +++ b/grafana/dashboards/postgresql/Sonarqube.json @@ -0,0 +1,1363 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 19, + "links": [ + { + "targetBlank": true, + "title": "SonarQube", + "url": "https://devlake.apache.org/docs/Plugins/sonarqube" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the code quality metrics from SonarQube.\n- Data Source Required: SonarQube v8.2+\n- Snapshot panels (Reliability, Security, Test, etc.) show the latest scan and do not honor the time filter.\n- **Historical Trends** panels at the bottom respond to the time range selector and show metric history over time.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 16, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Reliability & Security", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bugs" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Bugs\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'BUG' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Reliability", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "vulnerabilities" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 3, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Vulnerabilities\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'VULNERABILITY' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Security", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "hotspots" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Security Hotspots\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'HOTSPOTS' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Security Hotspots", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(CASE WHEN status <> 'TO_REVIEW' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) * 100 AS DECIMAL), 2) || '%' AS \"Reviewed\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'HOTSPOTS' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Security Reviewed", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Test & Maintainability", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Test", + "url": "https://devlake.apache.org/docs/Metrics/CQTest" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST((SUM(lines_to_cover) - SUM(uncovered_lines)) AS NUMERIC) / NULLIF(SUM(lines_to_cover), 0) * 100, 1) || '% ' || 'Coverage on ' || ROUND(CAST(CAST(SUM(lines_to_cover) AS NUMERIC) / NULLIF(1000, 0) AS DECIMAL), 0) || 'k Lines to cover' FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Test", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 7, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Maintainability-Debt", + "url": "https://devlake.apache.org/docs/Metrics/CQMaintainability-Debt" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT FLOOR(CAST(CAST(SUM(debt) AS NUMERIC) / NULLIF(8, 0) AS NUMERIC) / NULLIF(60, 0)) || ' day(s) ' || FLOOR(CAST((SUM(debt) % 480) AS NUMERIC) / NULLIF(60, 0)) || ' hour(s) ' AS \"Debt\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'CODE_SMELL' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Maintainability - Debt", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "code smells" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 14, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Maintainability-Debt", + "url": "https://devlake.apache.org/docs/Metrics/CQMaintainability-Debt" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Code Smells\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'CODE_SMELL' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Maintainability - Code Smells", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 6, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Duplication & Code", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Blocks", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedBlocks" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(duplicated_blocks) FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duplicated Blocks", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Lines", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedLines" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(duplicated_lines) AS NUMERIC) / NULLIF(SUM(num_of_lines), 0) * 100, 1) || '% ' || 'Duplications on ' || ROUND(CAST(CAST(SUM(ncloc) AS NUMERIC) / NULLIF(1000, 0) AS DECIMAL), 0) || 'k Lines' FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duplicated Lines", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-text" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 17, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT file_name, num_of_lines AS \"Lines of Code\", bugs AS \"Bugs\", vulnerabilities AS \"Vulnerabilities\", code_smells AS \"Code Smells\", security_hotspots AS \"Security Hotspots\", ROUND(coverage, 2) || '%' AS \"Coverage\", ROUND(duplicated_lines_density, 2) || '%' AS \"Duplications\" FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) ORDER BY bugs DESC NULLS LAST LIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Code Quality Metrics by Files (Top 20 order by Bugs)", + "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 20, + "panels": [ + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.coverage AS \"Coverage %\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Coverage Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.bugs AS \"Bugs\", pmh.vulnerabilities AS \"Vulnerabilities\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Bugs & Vulnerabilities Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.code_smells AS \"Code Smells\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Code Smells Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.duplicated_lines_density AS \"Duplication %\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Duplication Trend", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Historical Trends", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM cq_projects", + "hide": 0, + "includeAll": true, + "label": "SonarQube Project", + "multi": true, + "name": "project_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cq_projects", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT severity FROM cq_issues WHERE id::text LIKE 'sonar%'", + "hide": 0, + "includeAll": true, + "label": "Severity", + "multi": true, + "name": "severity", + "options": [], + "query": "SELECT DISTINCT severity FROM cq_issues WHERE id::text LIKE 'sonar%'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "SonarQube Server", + "uid": "WA0qbuJ4k-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/TAPD.json b/grafana/dashboards/postgresql/TAPD.json new file mode 100644 index 00000000000..273f1dafe10 --- /dev/null +++ b/grafana/dashboards/postgresql/TAPD.json @@ -0,0 +1,1195 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 13, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "TAPD", + "url": "https://devlake.apache.org/docs/Plugins/tapd" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from TAPD.\n- Data Source Required: TAPD", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'tapd%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'tapd%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "TAPD", + "uid": "hi-907hVk-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Taiga.json b/grafana/dashboards/postgresql/Taiga.json new file mode 100644 index 00000000000..39bd23a5e82 --- /dev/null +++ b/grafana/dashboards/postgresql/Taiga.json @@ -0,0 +1,2442 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [ + { + "targetBlank": true, + "title": "Taiga Plugin", + "url": "https://github.com/apache/incubator-devlake" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows project management metrics from Taiga across all issue types (User Stories, Tasks, Issues/Requirements, Epics).\n- Data Source Required: Taiga\n- Metrics: Issue throughput (filtered by type), lead time, story points (User Stories), milestone progress, assignee breakdown and issue type breakdown.\n- Use the **Issue Type** variable to filter panels to a specific type, or select All to see all types together.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 100, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total number of user stories created in the selected time range and board.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total number of issues that have been marked as Done/Closed in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 3, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Closed Issues [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Monthly distribution of open vs. closed issues.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Stories\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Closed Stories\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Closed Issues) / count(All Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 5, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Monthly trend of issue delivery rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 101, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average number of days from story creation to resolution.", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 7, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Lead Time in Days [Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "80% of stories are resolved within this many days.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Stories' Lead Time \u2264 # Days [Resolved in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Mean lead time per month for resolved stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time (days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Lead Time by Month [Resolved in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "The cumulative distribution of story lead time. Each point shows the percent rank of a given lead time value.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.51, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Cumulative Distribution of Lead Time [Resolved in Selected Time Range]", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "current" + ] + } + } + ], + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 102, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "3. Story Points", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total story points for all user stories in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 30 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT SUM(ts.total_points) AS value FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Story Points", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total story points for closed/completed user stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 30 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT SUM(ts.total_points) AS value FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id JOIN issues AS i ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.status = 'DONE'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Completed Story Points", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Story points allocated to each milestone/sprint. Requires milestone data from Taiga.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Story Points", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 30 + }, + "id": 13, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(ts.milestone_name, ''), 'No Milestone') AS \"Milestone\", SUM(ts.total_points) AS \"Story Points\" FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY \"Story Points\" DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points by Milestone", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Story points distributed across assignees.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Story Points", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 14, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(i.assignee_name, ''), 'Unassigned') AS \"Assignee\", SUM(i.story_point) AS \"Story Points\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY \"Story Points\" DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points by Assignee [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of story points across open vs. closed stories.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 15, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT SUM(ts.total_points) AS \"Story Points\", CASE WHEN i.status = 'DONE' THEN 'Completed' ELSE 'In Progress' END AS \"Status\" FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id JOIN issues AS i ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY \"Status\"", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points: Completed vs In Progress", + "type": "piechart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 103, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "4. Issues Overview", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Breakdown of all Taiga issues by their current status, across all issue types (User Story, Task, Issue/Requirement, Epic).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 43 + }, + "id": 16, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT NOW() AS time, COUNT(DISTINCT CASE WHEN i.status = 'TODO' THEN i.id END) AS \"To Do\", COUNT(DISTINCT CASE WHEN i.status = 'IN_PROGRESS' THEN i.id END) AS \"In Progress\", COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id END) AS \"Done\", COUNT(DISTINCT CASE WHEN NOT i.status IN ('TODO', 'IN_PROGRESS', 'DONE') THEN i.id END) AS \"Other\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Issues by Status", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of user stories assigned to each team member.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "User Stories", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 43 + }, + "id": 17, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(i.assignee_name, ''), 'Unassigned') AS \"Assignee\", COUNT(DISTINCT i.id) AS \"User Stories\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY \"User Stories\" DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issues by Assignee [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of user stories in each milestone/sprint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "User Stories", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 43 + }, + "id": 18, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(ts.milestone_name, ''), 'No Milestone') AS \"Milestone\", COUNT(DISTINCT ts.user_story_id) AS \"User Stories\" FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY \"User Stories\" DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "User Stories per Milestone", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Milestone progress: open vs. closed user stories per sprint.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 19, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": -15, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(ts.milestone_name, ''), 'No Milestone') AS \"Milestone\", COUNT(DISTINCT CASE WHEN i.status <> 'DONE' THEN ts.user_story_id ELSE NULL END) AS \"Open Stories\", COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN ts.user_story_id ELSE NULL END) AS \"Closed Stories\" FROM _tool_taiga_user_stories AS ts JOIN board_issues AS bi ON bi.issue_id = 'taiga:TaigaUserStory:' || ts.connection_id || ':' || ts.user_story_id JOIN issues AS i ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY \"Closed Stories\" DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Milestone Progress: Open vs. Closed Stories", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 104, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "5. Issue Type Breakdown", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of all Taiga items across issue types (User Story, Task, Issue/Requirement, Epic).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 61 + }, + "id": 20, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT NOW() AS time, COUNT(DISTINCT CASE WHEN i.type = 'USER_STORY' THEN i.id END) AS \"User Story\", COUNT(DISTINCT CASE WHEN i.type = 'TASK' THEN i.id END) AS \"Task\", COUNT(DISTINCT CASE WHEN i.type = 'REQUIREMENT' THEN i.id END) AS \"Issue\", COUNT(DISTINCT CASE WHEN i.type = 'EPIC' THEN i.id END) AS \"Epic\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Issues by Type", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Open vs closed count for each Taiga issue type in the selected time range.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 6, + "y": 61 + }, + "id": 21, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(i.original_type, i.type) AS \"Issue Type\", COUNT(DISTINCT CASE WHEN i.status <> 'DONE' THEN i.id ELSE NULL END) AS \"Open\", COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS \"Closed\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Open vs Closed by Type [Created in Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Monthly count of items created, broken down by issue type (User Stories, Tasks, Issues, Epics).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 61 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN i.type = 'USER_STORY' THEN i.id END) AS \"User Stories\", COUNT(DISTINCT CASE WHEN i.type = 'TASK' THEN i.id END) AS \"Tasks\", COUNT(DISTINCT CASE WHEN i.type = 'REQUIREMENT' THEN i.id END) AS \"Issues\", COUNT(DISTINCT CASE WHEN i.type = 'EPIC' THEN i.id END) AS \"Epics\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND $__timeFilter(i.created_date) GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Items Created per Month by Type", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average lead time in days from creation to resolution, broken down by issue type (resolved issues only).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Avg Lead Time (days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 23, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(i.original_type, i.type) AS \"Issue Type\", ROUND(CAST(AVG(CAST(COALESCE(i.lead_time_minutes, (EXTRACT(EPOCH FROM (i.resolution_date - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0)) AS DECIMAL), 1) AS \"Avg Lead Time (days)\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.status = 'DONE' AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Average Lead Time by Type", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'taiga%'", + "hide": 0, + "includeAll": true, + "label": "Choose Project", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'taiga%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT i.type FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE bi.board_id::text LIKE 'taiga%'", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": true, + "name": "type", + "options": [], + "query": "SELECT DISTINCT i.type FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE bi.board_id::text LIKE 'taiga%'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Taiga", + "uid": "taiga-dashboard-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Teambition.json b/grafana/dashboards/postgresql/Teambition.json new file mode 100644 index 00000000000..2b0d2e62e3d --- /dev/null +++ b/grafana/dashboards/postgresql/Teambition.json @@ -0,0 +1,1195 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Teambition", + "url": "https://devlake.apache.org/docs/Plugins/teambition" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Teambition.\n- Data Source Required: Teambition", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'teambition%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'teambition%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Teambition", + "uid": "hi-908hVk-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Testmo.json b/grafana/dashboards/postgresql/Testmo.json new file mode 100644 index 00000000000..86cdf283996 --- /dev/null +++ b/grafana/dashboards/postgresql/Testmo.json @@ -0,0 +1,1232 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Testmo", + "url": "https://www.testmo.com/" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows comprehensive test management metrics from Testmo.\n- Data Source Required: Testmo\n- Metrics: Test execution, success rates, coverage, and performance analytics", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Test Execution Overview", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total number of test runs executed in the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS value FROM (SELECT DISTINCT ar.id, ar.connection_id, ar.testmo_created_at FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT DISTINCT r.id, r.connection_id, r.testmo_created_at FROM _tool_testmo_runs AS r) AS combined WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Total Test Runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage of test runs that passed successfully", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "green", + "value": 90 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(COUNT(CASE WHEN combined.status IN (1, 2) THEN 1 ELSE NULL END) * 1.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS value FROM (SELECT ar.id, ar.connection_id, ar.testmo_created_at, ar.status FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT r.id, r.connection_id, r.testmo_created_at, r.status FROM _tool_testmo_runs AS r) AS combined WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Test Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Test execution results over time showing pass/fail/skip distribution", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Passed Tests" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "green" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed Tests" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Skipped Tests" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "yellow" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(combined.testmo_created_at AS DATE) AS time, COUNT(CASE WHEN combined.status = 1 THEN 1 ELSE NULL END) AS \"Passed Tests\", COUNT(CASE WHEN combined.status = 2 THEN 1 ELSE NULL END) AS \"Failed Tests\", COUNT(CASE WHEN NOT combined.status IN (1, 2) THEN 1 ELSE NULL END) AS \"Other Tests\" FROM (SELECT ar.id, ar.connection_id, ar.testmo_created_at, ar.status FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT r.id, r.connection_id, r.testmo_created_at, r.status FROM _tool_testmo_runs AS r) AS combined WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(combined.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Test Execution Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total number of Testmo runs (new data source)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 10 + }, + "id": 117, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT r.id) AS value FROM _tool_testmo_runs AS r WHERE $__timeFilter(r.testmo_created_at) AND ('${connection_id:csv}' = '' OR r.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Testmo Runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average execution time per test run in minutes", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 10 + }, + "id": 118, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT ar.id) AS value FROM _tool_testmo_automation_runs AS ar WHERE $__timeFilter(ar.testmo_created_at) AND ('${connection_id:csv}' = '' OR ar.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "refId": "A" + } + ], + "title": "Test Runs in Period", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily count of test runs from the new runs data source", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Number of Runs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short", + "decimals": 0 + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Test Runs" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 121, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(r.testmo_created_at AS DATE) AS time, COUNT(*) AS \"Test Runs\" FROM _tool_testmo_runs AS r WHERE $__timeFilter(r.testmo_created_at) AND ('${connection_id:csv}' = '' OR r.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(r.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Daily Test Runs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Test coverage across different projects", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Number of Tests", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 122, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT p.name AS metric, COUNT(*) AS \"Test Runs\" FROM (SELECT ar.id, ar.connection_id, ar.testmo_created_at, ar.project_id FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT r.id, r.connection_id, r.testmo_created_at, r.project_id FROM _tool_testmo_runs AS r) AS combined JOIN _tool_testmo_projects AS p ON combined.project_id = p.id AND combined.connection_id = p.connection_id WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY p.name ORDER BY COUNT(*) DESC NULLS LAST LIMIT 10", + "refId": "A" + } + ], + "title": "Tests by Project", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Test Performance & Quality", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Success rate trend over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Success Rate (%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 123, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(combined.testmo_created_at AS DATE) AS time, CAST(COUNT(CASE WHEN combined.status IN (1, 2) THEN 1 ELSE NULL END) * 1.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Success Rate\" FROM (SELECT ar.id, ar.connection_id, ar.testmo_created_at, ar.status FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT r.id, r.connection_id, r.testmo_created_at, r.status FROM _tool_testmo_runs AS r) AS combined WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(combined.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Test Success Rate Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average test execution time trend", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Duration (minutes)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 124, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(ar.testmo_created_at AS DATE) AS time, COUNT(DISTINCT ar.id) AS \"Test Count\" FROM _tool_testmo_automation_runs AS ar WHERE $__timeFilter(ar.testmo_created_at) AND ('${connection_id:csv}' = '' OR ar.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(ar.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Test Count Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Comparison between automation runs and regular test runs over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Number of Runs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 8, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Automation Runs" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Test Runs" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 131, + "options": { + "legend": { + "calcs": [ + "sum", + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(ar.testmo_created_at AS DATE) AS time, COUNT(DISTINCT ar.id) AS \"Automation Runs\" FROM _tool_testmo_automation_runs AS ar WHERE $__timeFilter(ar.testmo_created_at) AND ('${connection_id:csv}' = '' OR ar.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(ar.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "A" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(r.testmo_created_at AS DATE) AS time, COUNT(DISTINCT r.id) AS \"Test Runs\" FROM _tool_testmo_runs AS r WHERE $__timeFilter(r.testmo_created_at) AND ('${connection_id:csv}' = '' OR r.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) GROUP BY CAST(r.testmo_created_at AS DATE) ORDER BY time NULLS FIRST", + "refId": "B" + } + ], + "title": "Automation Runs vs Test Runs Comparison", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Test runs that take the longest time to execute", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "m" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Duration" + }, + "properties": [ + { + "id": "unit", + "value": "m" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 125, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT combined.id AS \"Test Run ID\", p.name AS \"Project\", combined.name AS \"Test Name\", combined.status_name AS \"Status\", combined.testmo_created_at AS \"Created Date\", combined.source AS \"Source\" FROM (SELECT ar.id, ar.connection_id, ar.testmo_created_at, ar.project_id, ar.name, CASE WHEN ar.status = 1 THEN 'Failed' WHEN ar.status = 2 THEN 'Passed' ELSE 'Other' END AS status_name, 'Automation Run' AS source FROM _tool_testmo_automation_runs AS ar UNION ALL SELECT r.id, r.connection_id, r.testmo_created_at, r.project_id, r.name, r.status_name, 'Test Run' AS source FROM _tool_testmo_runs AS r) AS combined JOIN _tool_testmo_projects AS p ON combined.project_id = p.id AND combined.connection_id = p.connection_id WHERE $__timeFilter(combined.testmo_created_at) AND ('${connection_id:csv}' = '' OR combined.connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[])) ORDER BY combined.testmo_created_at DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "Recent Test Runs", + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM _tool_testmo_connections", + "hide": 0, + "includeAll": true, + "label": "Choose Connection", + "multi": true, + "name": "connection_id", + "options": [], + "query": "SELECT name || '--' || id FROM _tool_testmo_connections", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM _tool_testmo_projects WHERE ('${connection_id:csv}' = '' OR connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM _tool_testmo_projects WHERE ('${connection_id:csv}' = '' OR connection_id::text = ANY(ARRAY[${connection_id:singlequote}]::text[]))", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Testmo", + "uid": "testmo-dashboard-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/Zentao.json b/grafana/dashboards/postgresql/Zentao.json new file mode 100644 index 00000000000..046f8bf7928 --- /dev/null +++ b/grafana/dashboards/postgresql/Zentao.json @@ -0,0 +1,1195 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 27, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "Zentao", + "url": "https://devlake.apache.org/docs/Plugins/zentao" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic project management metrics from Zentao.\n- Data Source Required: Zentao", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Issue Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of issues created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Issues [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Delivered Issue [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 12, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS \"Number of Open Issues\", COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS \"Number of Delivered Issues\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Status Distribution over Month [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS delivered_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * delivered_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate [Issues Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Issue Delivery Rate = count(Delivered Issues)/count(Issues)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Delivery Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS delivered_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, delivered_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Delivery Rate over Time [Issues Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Lead Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Issues' Lead Time are less than # days [Issues Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_lead_time FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time [Issues Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of requirement lead time. \n2. Each point refers to the percent rank of a lead time.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of Requirement Lead Time [Issues Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'zentao%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'zentao%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Zentao", + "uid": "yMb4MKh4k-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/ai-cost-efficiency.json b/grafana/dashboards/postgresql/ai-cost-efficiency.json new file mode 100644 index 00000000000..497a05b0f34 --- /dev/null +++ b/grafana/dashboards/postgresql/ai-cost-efficiency.json @@ -0,0 +1,537 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits Used", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0) AS DECIMAL), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per PR (Overall)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0) AS DECIMAL), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Deployment (Overall)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0) AS DECIMAL), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly cost per merged PR", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Merged PR (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly cost per production deployment", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Deployment (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly cost per resolved issue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved (Weekly)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Is cost efficiency improving over time?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly AI Activity Volume", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "cost", + "efficiency" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "AI Cost-Efficiency", + "uid": "ai_cost_efficiency-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/ai-model-roi.json b/grafana/dashboards/postgresql/ai-model-roi.json new file mode 100644 index 00000000000..ffbda08b227 --- /dev/null +++ b/grafana/dashboards/postgresql/ai-model-roi.json @@ -0,0 +1,537 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Summary", + "type": "row" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(SUM(credits_used)) AS \"Total Credits\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits Used", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT pr.id), 0), 0), 0), 1) AS \"Credits / PR\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date)) AS pr WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per PR (Overall)", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT cdc.cicd_deployment_id), 0), 0), 0), 1) AS \"Credits / Deploy\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT cicd_deployment_id FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date)) AS cdc WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Deployment (Overall)", + "type": "stat" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS DECIMAL) / NULLIF(NULLIF(NULLIF(COUNT(DISTINCT i.id), 0), 0), 0), 1) AS \"Credits / Issue\" FROM _tool_q_dev_user_report AS r CROSS JOIN (SELECT DISTINCT id FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date)) AS i WHERE $__timeFilter(r.date)", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per merged PR", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _prs AS (SELECT CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS prs FROM pull_requests WHERE NOT merged_date IS NULL AND $__timeFilter(merged_date) GROUP BY CAST(merged_date AS DATE) - (EXTRACT(ISODOW FROM merged_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(p.prs, 0), 0), 1) AS \"Credits per PR\" FROM _credits AS c LEFT JOIN _prs AS p ON c.week_start = p.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Merged PR (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per production deployment", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _deploys AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cicd_deployment_id) AS deploys FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM finished_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(d.deploys, 0), 0), 1) AS \"Credits per Deploy\" FROM _credits AS c LEFT JOIN _deploys AS d ON c.week_start = d.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Deployment (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Weekly cost per resolved issue", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _credits AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day'), _issues AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS resolved FROM issues WHERE NOT resolution_date IS NULL AND type <> 'INCIDENT' AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM resolution_date) - 1) * INTERVAL '1 day') SELECT c.week_start AS time, ROUND(CAST(c.credits AS NUMERIC) / NULLIF(NULLIF(i.resolved, 0), 0), 1) AS \"Credits per Issue\" FROM _credits AS c LEFT JOIN _issues AS i ON c.week_start = i.week_start ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits per Issue Resolved (Weekly)", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "description": "Is cost efficiency improving over time?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": "postgresql", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM date) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly AI Activity Volume", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "cost", + "efficiency" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "AI Cost-Efficiency (PostgreSQL)", + "uid": "ai_cost_efficiency-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/argo-cd.json b/grafana/dashboards/postgresql/argo-cd.json new file mode 100644 index 00000000000..523b012ddc0 --- /dev/null +++ b/grafana/dashboards/postgresql/argo-cd.json @@ -0,0 +1,1033 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [ + { + "targetBlank": true, + "title": "ArgoCD", + "url": "https://devlake.apache.org/docs/Plugins/argocd" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: Track ArgoCD deployment activity, measure success rates, monitor environments, and spot slow deployments.\n- Data Source Required: ArgoCD plugin (deployments mapped into `cicd_deployments`).\n- Filters: Use the *Application* templating variable to focus on specific ArgoCD applications.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ',')))", + "refId": "A" + } + ], + "title": "1.1 Total Deployments", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ',')))", + "refId": "A" + } + ], + "title": "1.2 Successful Deployments", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Successful deployments / Total deployments", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT id), 0), 0) FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ',')))", + "refId": "A" + } + ], + "title": "1.3 Deployment Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 5, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT result, COUNT(DISTINCT id) AS deployment_count FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A" + } + ], + "title": "1.4 Deployment Result Distribution", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "title": "2. Deployment Trends", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _deployments AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COUNT(DISTINCT id) AS deployment_count FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, deployment_count FROM _deployments ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "2.1 Deployments per Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _deployment_success_rate AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, result, id FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) GROUP BY 1, 2, 3) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Deployment Success Rate\" FROM _deployment_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "2.2 Deployment Success Rate by Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 15, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT environment, COUNT(DISTINCT id) AS deployment_count FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A" + } + ], + "title": "2.3 Deployments by Environment", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 9, + "panels": [], + "title": "3. Deployment Durations", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average duration of deployments (minutes)", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) AND NOT duration_sec IS NULL", + "refId": "A" + } + ], + "title": "3.1 Mean Deployment Duration (Minutes)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 19 + }, + "id": 11, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _durations AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) AND NOT duration_sec IS NULL GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(mean_duration_sec AS NUMERIC) / NULLIF(60, 0) AS mean_duration_minutes FROM _durations ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "3.2 Mean Deployment Duration by Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 19 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration_minutes" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT name AS deployment_name, display_title AS description, environment, result, CAST(duration_sec AS NUMERIC) / NULLIF(60, 0) AS duration_minutes, finished_date FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) AND NOT duration_sec IS NULL ORDER BY duration_sec DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "3.3 Top 20 Longest Deployments", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "7-day rolling average of deployment duration (minutes)", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH daily AS (SELECT CAST(created_date AS DATE) AS day_bucket, AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS avg_duration_minutes FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) AND NOT duration_sec IS NULL GROUP BY 1), rolling AS (SELECT day_bucket, AVG(avg_duration_minutes) OVER (ORDER BY day_bucket NULLS FIRST ROWS BETWEEN 6 PRECEDING AND CURRENT ROW) AS rolling_avg_minutes FROM daily) SELECT day_bucket AS time, rolling_avg_minutes FROM rolling ORDER BY day_bucket NULLS FIRST", + "refId": "A" + } + ], + "title": "3.4 Rolling Deployment Duration (7-day Avg)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 13, + "panels": [], + "title": "4. Deployment Details", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "created_date" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT created_date, name AS deployment_name, environment, result, status, display_title AS description, finished_date FROM cicd_deployments WHERE $__timeFilter(created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) ORDER BY created_date DESC NULLS LAST LIMIT 50", + "refId": "A" + } + ], + "title": "4.1 Recent Deployments", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 17, + "panels": [], + "title": "5. Images", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "deployment_created" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT d.created_date AS deployment_created, d.name AS deployment_name, ri.images::text AS images, c.commit_sha AS revision, d.environment, d.result FROM cicd_deployments AS d LEFT JOIN cicd_deployment_commits AS c ON c.cicd_deployment_id = d.id LEFT JOIN _tool_argocd_revision_images AS ri ON ri.revision = c.commit_sha WHERE $__timeFilter(d.created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR d.cicd_scope_id = ANY(string_to_array('${application_id:raw}', ','))) ORDER BY d.created_date DESC NULLS LAST LIMIT 50", + "refId": "A" + } + ], + "title": "5.1 Recent Deployment Images", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Approximation: counts deployments per unique images array (revision-level). For per-image counts, consider exploding arrays during ingestion.", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 19, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH dep AS (SELECT d.id, c.commit_sha, d.cicd_scope_id FROM cicd_deployments AS d LEFT JOIN cicd_deployment_commits AS c ON c.cicd_deployment_id = d.id WHERE $__timeFilter(d.created_date) AND ('${application_id:raw}' = '' OR '${application_id:raw}' = '$__all' OR d.cicd_scope_id = ANY(string_to_array('${application_id:raw}', ',')))), imgs AS (SELECT dep.id AS deployment_id, ri.images::text FROM dep LEFT JOIN _tool_argocd_revision_images AS ri ON ri.revision = dep.commit_sha WHERE NOT ri.images::text IS NULL) SELECT images AS image_array, COUNT(DISTINCT deployment_id) AS deployment_count FROM imgs GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "5.2 Top Image Arrays (Approx)", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "allValue": "$__all", + "definition": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'argocd:ArgocdApplication:%'", + "hide": 0, + "includeAll": true, + "label": "Application", + "multi": true, + "name": "application_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE 'argocd:ArgocdApplication:%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "ArgoCD", + "uid": "Argocd001-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/azure-dev-ops.json b/grafana/dashboards/postgresql/azure-dev-ops.json new file mode 100644 index 00000000000..85d27bca014 --- /dev/null +++ b/grafana/dashboards/postgresql/azure-dev-ops.json @@ -0,0 +1,2335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 20, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 101, + "links": [ + { + "targetBlank": true, + "title": "Azure DevOps", + "url": "https://devlake.apache.org/docs/Plugins/azuredevops" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from Azure DevOps.\n- Data Source Required: Azure DevOps", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 83, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Contribution (PRs)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 68, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Number of New Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pull Request Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 4 + }, + "id": 77, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT id) AS pr_count FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(created_date) /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, pr_count AS \"Pull Request Count\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.2 Number of New Pull Requests", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Merged PR Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 54, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT author_name, COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'MERGED' GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.3 Top Contributors By Merged PRs", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 85, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. How PRs are handled?", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 66, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('MERGED', 'CLOSED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Ratio of unmerged Pull Requests of All Closed or Merged PRs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Open" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed without merging" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed and merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 17 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status = 'OPEN' THEN id ELSE NULL END) AS open, COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS closed, COUNT(DISTINCT CASE WHEN status = 'MERGED' THEN id ELSE NULL END) AS merged FROM pull_requests WHERE $__timeFilter(created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, open AS \"PR: Open\", closed AS \"PR: Closed without merging\", merged AS \"PR: Closed and merged\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.2 Pull Request Status Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 80, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'CLOSED'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.3 Number of Pull Requests Closed without Merging [Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ratio", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 23 + }, + "id": 81, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('MERGED', 'CLOSED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.4 Ratio of unmerged PRs of All Closed or Merged PRs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 72, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.5 Mean Time to Merge of Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 29 + }, + "id": 95, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_merge FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, time_to_merge AS \"Time to Merge\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.6 Mean Time to Merge of Pull Requests", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 102, + "panels": [], + "title": "3. CI/CD Metrics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of Pipeline runs executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 103, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of Successful Pipeline runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful Pipeline runs / Number of total Pipeline runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 104, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.2 Mean Pipeline runs Success Rate %", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 110, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.25, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, \"successful_count\", failed_count FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.3 Number of successful and failed pipeline runs", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful Pipeline runs / Number of total Pipeline runs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 105, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN result = '' THEN 'Unknown' ELSE result END AS result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.4 Total Pipeline runs Result Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 42 + }, + "id": 112, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND /* and id::text LIKE '%azure%' */ ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY \"time\", \"result\", id) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN TO_CHAR(CAST(time AS TIMESTAMP), 'MM/YYYY') ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Pipeline Runs Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.5 Pipeline run success rate %", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Pipeline runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"Pipeline runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 106, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.6 Mean Pipeline runs Duration in Minutes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_sec" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 48 + }, + "id": 111, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND /* and id::text LIKE '%azure%' */ ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, mean_duration_sec FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.7 Mean pipeline run duration", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 99, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'azure%'", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'azure%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + "hide": 0, + "includeAll": false, + "label": "Time Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Azure DevOps", + "uid": "ba7e3a95-80ed-4067-a54b-2a82758eb3dd-pg", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/bit-bucket.json b/grafana/dashboards/postgresql/bit-bucket.json new file mode 100644 index 00000000000..91504944776 --- /dev/null +++ b/grafana/dashboards/postgresql/bit-bucket.json @@ -0,0 +1,2335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 37, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 101, + "links": [ + { + "targetBlank": true, + "title": "BitBucket Cloud", + "url": "https://devlake.apache.org/docs/Plugins/bitbucket" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from Bitbucket.\n- Data Source Required: Bitbucket Cloud or Server/data center. Please note that for Bitbucket Server/Data center, there is no embedded CI/CD data.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 83, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Contribution (PRs)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 68, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Number of New Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pull Request Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 4 + }, + "id": 77, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT id) AS pr_count FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, pr_count AS \"Pull Request Count\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.2 Number of New Pull Requests", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Merged PR Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 54, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 59, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT author_name, COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'MERGED' GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.3 Top 20 Contributors By Merged PRs", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 85, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. How PRs are handled?", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 17 + }, + "id": 66, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('MERGED', 'CLOSED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Ratio of unmerged Pull Requests of All Closed or Merged PRs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Open" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed without merging" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed and merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 17 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status = 'OPEN' THEN id ELSE NULL END) AS open, COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS closed, COUNT(DISTINCT CASE WHEN status = 'MERGED' THEN id ELSE NULL END) AS merged FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, open AS \"PR: Open\", closed AS \"PR: Closed without merging\", merged AS \"PR: Closed and merged\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.2 Pull Request Status Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 80, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'CLOSED'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.3 Number of Pull Requests Closed without Merging [Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ratio", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 23 + }, + "id": 81, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('MERGED', 'CLOSED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.4 Ratio of Non-merging PRs of All Closed or Merged PRs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 72, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.5 Mean Time to Merge of Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 29 + }, + "id": 95, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_merge FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, time_to_merge AS \"Time to Merge\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.6 Mean Time to Merge of Pull Requests", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 109, + "panels": [], + "title": "CI/CD Metrics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of pipeline runs executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 102, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of successful pipeline runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful pipeline runs / Number of total pipeline runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 103, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* Enable the following condition to remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.2 Mean pipeline run success rate %", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 104, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.25, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, \"successful_count\", failed_count FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.3 Number of successful and failed pipeline runs", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 105, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN result = '' THEN 'Unknown' ELSE result END AS result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.4 Pipeline runs result distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Pipeline Runs success rate over time.\n2. The Pipeline runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 42 + }, + "id": 106, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY \"time\", \"result\", id) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN TO_CHAR(CAST(time AS TIMESTAMP), 'MM/YYYY') ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Pipeline Runs Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.5 Pipeline run success rate %", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful pipeline runs / Number of total pipeline runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 107, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* Enable the following condition to remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.6 Mean pipeline runs duration", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_sec" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 48 + }, + "id": 108, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE 'bitbucket%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, mean_duration_sec FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.7 Mean pipeline run duration", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 99, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'bitbucket%'", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'bitbucket%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + "hide": 0, + "includeAll": false, + "label": "Time Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Bitbucket", + "uid": "4LzQHZa4k-pg", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/circle-ci.json b/grafana/dashboards/postgresql/circle-ci.json new file mode 100644 index 00000000000..16b0be85ba7 --- /dev/null +++ b/grafana/dashboards/postgresql/circle-ci.json @@ -0,0 +1,1272 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 27, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 58, + "links": [ + { + "targetBlank": true, + "title": "CircleCI", + "url": "https://devlake.apache.org/docs/Plugins/circleci" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic CI/CD metrics from CircleCI, such as [Build Count](https://devlake.apache.org/docs/Metrics/BuildCount), [Build Duration](https://devlake.apache.org/docs/Metrics/BuildDuration) and [Build Success Rate](https://devlake.apache.org/docs/Metrics/BuildSuccessRate).\n- Data Source Required: CircleCI", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of builds executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Total Number of Successful Workflows [Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful workflows / Number of total workflows", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. Mean Workflow Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 37, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[])) GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Total Workflow Result Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful workflows / Number of total workflows", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 55, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4. Mean Workflow Duration in Minutes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 52, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, build_count AS \"Workflow Count\" FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Total Number of Successful Workflows [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean workflows success rate over time.\n2. The workflows being calculated are filtered by \"build starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Workflow Success Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Workflow Success Rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 50, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[])) GROUP BY \"time\", \"result\", id) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Workflow Success Rate\" FROM _build_success_rate GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Workflow Success Rate [Each Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_build_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 54, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_workflow_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_workflow_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of Successful and Failed Workflows [Each Month]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Build Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_minutes" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 56, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(finished_date AS DATE)) + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%circleci%' AND ('${full_name:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${full_name:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, CAST(mean_duration_sec AS NUMERIC) / NULLIF(60, 0) AS mean_duration_minutes FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.1 Mean Workflow Duration in Minutes [Each Month]", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 60, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE '%circleci%'", + "hide": 0, + "includeAll": true, + "label": "CircleCI Project", + "multi": true, + "name": "full_name", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cicd_scopes WHERE id::text LIKE '%circleci%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "CircleCI", + "uid": "CircleCI-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/component-and-file-level-metrics.json b/grafana/dashboards/postgresql/component-and-file-level-metrics.json new file mode 100644 index 00000000000..f89723ae82f --- /dev/null +++ b/grafana/dashboards/postgresql/component-and-file-level-metrics.json @@ -0,0 +1,1443 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "This dashboard shows the metrics collected by gitextractor plugin", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 40, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Author dimension", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 16, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT author_name, COUNT(DISTINCT commit_sha) AS commit_nums FROM commits, repo_commits WHERE commits.sha = repo_commits.commit_sha AND ('${repo_id:csv}' = '' OR repo_commits.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(commits.authored_date) GROUP BY \"author_name\", \"author_id\" ORDER BY commit_nums DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "commits distribution by author", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 79, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT author_name, SUM(additions - deletions) AS cnt FROM commits, repo_commits WHERE commits.sha = repo_commits.commit_sha AND ('${repo_id:csv}' = '' OR repo_commits.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(commits.authored_date) GROUP BY \"author_name\", \"author_id\" ORDER BY cnt DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "lines of code distribution by author", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 20, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Time dimension", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 22, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CASE CAST(EXTRACT(ISODOW FROM CAST(authored_date AS DATE)) AS TEXT) WHEN '2' THEN '1.\"Monday\"' WHEN '3' THEN '2.\"Tuesday\"' WHEN '4' THEN '3.\"Wednesday\"' WHEN '5' THEN '4.\"Thursday\"' WHEN '6' THEN '5.\"Friday\"' WHEN '7' THEN '6.\"Saturday\"' WHEN '1' THEN '7.\"Sunday\"' END AS weekday, COUNT(DISTINCT commit_sha) AS commit_nums FROM commits, repo_commits WHERE $__timeFilter(commits.authored_date) AND commits.sha = repo_commits.commit_sha AND ('${repo_id:csv}' = '' OR repo_commits.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY \"weekday\"", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "commits distribution", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 24, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CASE CAST(EXTRACT(ISODOW FROM CAST(authored_date AS DATE)) AS TEXT) WHEN '2' THEN '1.\"Monday\"' WHEN '3' THEN '2.\"Tuesday\"' WHEN '4' THEN '3.\"Wednesday\"' WHEN '5' THEN '4.\"Thursday\"' WHEN '6' THEN '5.\"Friday\"' WHEN '7' THEN '6.\"Saturday\"' WHEN '1' THEN '7.\"Sunday\"' END AS weekday, SUM(additions - deletions) AS changed_nums, SUM(additions) AS total_additions, SUM(deletions) AS total_deletions FROM commits, repo_commits WHERE commits.sha = repo_commits.commit_sha AND ('${repo_id:csv}' = '' OR repo_commits.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(commits.authored_date) GROUP BY \"weekday\" ORDER BY weekday NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "lines of code distribution", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 14, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "File dimension", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 12, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.5, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT file_path, COUNT(DISTINCT c.author_name) AS cnt FROM commits AS c JOIN commit_files AS cf ON cf.commit_sha = c.sha JOIN repo_commits AS rc ON cf.commit_sha = rc.commit_sha WHERE ('${repo_id:csv}' = '' OR repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(c.authored_date) AND CAST(file_path AS TEXT) ~ '${selected_path:raw}' GROUP BY \"file_path\" ORDER BY cnt DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "_raw_data_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "issue_commits", + "timeColumn": "id", + "timeColumnType": "bigint", + "where": [] + } + ], + "title": "Files with maximum number of authors", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 2, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT file_path, COUNT(DISTINCT author_name) AS author_count, MAX(rst) AS lines_of_code, CAST(MAX(rst) AS NUMERIC) / NULLIF(COUNT(DISTINCT author_name), 0) AS rate FROM commits JOIN (SELECT file_path, commit_files.commit_sha, SUM(additions - deletions) AS rst FROM commit_files JOIN repo_commits AS rc ON commit_files.commit_sha = rc.commit_sha WHERE ('${repo_id:csv}' = '' OR repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND CAST(file_path AS TEXT) ~ '${selected_path:raw}' GROUP BY \"file_path\", commit_files.commit_sha) AS a ON a.commit_sha = commits.sha GROUP BY file_path HAVING author_count > 0 ORDER BY rate DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "[Selected Files] Largest files with lowest number of authors", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 10, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT file_path, COUNT(DISTINCT sha) AS modified_num FROM commits AS c JOIN commit_files AS cf ON cf.commit_sha = c.sha JOIN repo_commits AS rc ON cf.commit_sha = rc.commit_sha WHERE ('${repo_id:csv}' = '' OR repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(c.authored_date) AND CAST(file_path AS TEXT) ~ '${selected_path:raw}' GROUP BY \"file_path\" ORDER BY modified_num DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Files with most modifications", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT file_path, COUNT(DISTINCT sha) AS cnt FROM commits AS c JOIN commit_files AS cf ON cf.commit_sha = c.sha JOIN repo_commits AS rc ON cf.commit_sha = rc.commit_sha WHERE ('${repo_id:csv}' = '' OR repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(c.authored_date) GROUP BY \"file_path\" ORDER BY cnt DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "[All Files] Files with most modifications", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Lineage dimension", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 28, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.5, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CASE CAST(EXTRACT(ISODOW FROM CAST(commits.committed_date AS DATE)) AS TEXT) WHEN '2' THEN '1.\"Monday\"' WHEN '3' THEN '2.\"Tuesday\"' WHEN '4' THEN '3.\"Wednesday\"' WHEN '5' THEN '4.\"Thursday\"' WHEN '6' THEN '5.\"Friday\"' WHEN '7' THEN '6.\"Saturday\"' WHEN '1' THEN '7.\"Sunday\"' END AS wd, COUNT(*) AS lived_lines FROM repo_snapshot JOIN commits ON commits.sha = repo_snapshot.commit_sha WHERE ('${repo_id:csv}' = '' OR repo_snapshot.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(commits.committed_date) GROUP BY \"wd\" ORDER BY wd NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "In which day the code has highest chance to stay in repository.", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Please disable the SkipCommitFiles in the .env setting to view this metric. However, this will significantly increase the time to collect data.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 30, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT file_path, AVG((EXTRACT(EPOCH FROM (NOW() - commits.committed_date))/86400)) AS line_age FROM repo_snapshot JOIN commits ON repo_snapshot.commit_sha = commits.sha WHERE ('${repo_id:csv}' = '' OR repo_snapshot.repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(commits.committed_date) GROUP BY \"file_path\" ORDER BY line_age DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Which file has the longest average line age?", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM repos", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": ".*", + "value": ".*" + }, + "description": "input file path", + "hide": 0, + "label": "", + "name": "selected_path", + "options": [ + { + "selected": false, + "text": "", + "value": "" + } + ], + "query": ".*", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-5y", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "utc", + "title": "Component and File-level Metrics", + "uid": "KxUh7IG4z-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/contributor-experience.json b/grafana/dashboards/postgresql/contributor-experience.json new file mode 100644 index 00000000000..54d28fbb944 --- /dev/null +++ b/grafana/dashboards/postgresql/contributor-experience.json @@ -0,0 +1,930 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 7, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 24, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard answers the question \"What makes a great developer experience? And how can we define and track that?\". This dashboard heavily focuses on actionability. All metrics can be deterministically improved as long as OSS maintainers invested time into them.\n- Data Source Required: GitHub", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 8, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "titleSize": 2 + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT i.id AS issue_id, i.url, i.title, i.created_date AS issue_created_date, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))) SELECT AVG(CAST(((EXTRACT(EPOCH FROM (comment_date - issue_created_date))/60)) AS NUMERIC) / NULLIF(1440, 0)) FROM issue_comment_list WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Time To Initial Issue Response [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 4, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS issue_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND i.status = 'DONE' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Resolution Time [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 12, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT i.id AS issue_id, i.url, i.title, i.created_date AS issue_created_date, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))) SELECT CAST(100 * SUM(CASE WHEN CAST(((EXTRACT(EPOCH FROM (comment_date - issue_created_date))/60)) AS NUMERIC) / NULLIF(60, 0) < $iir_sla THEN 1 ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(*), 0) FROM issue_comment_list WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Response Rate within SLA [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 10, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN issue_labels AS il ON il.issue_id = i.id WHERE il.label_name = '$label_gfi' AND i.status <> 'DONE' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Good First Issues [Outstanding]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 16, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH pr_comment_list AS (SELECT pr.id AS issue_id, pr.url, pr.title, pr.created_date AS pr_created_date, prc.id AS comment_id, prc.created_date AS comment_date, prc.account_id, CASE WHEN NOT prc.id IS NULL THEN RANK() OVER (PARTITION BY pr.id ORDER BY prc.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id WHERE CAST(pr.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR pr.base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))) SELECT AVG(CAST(((EXTRACT(EPOCH FROM (comment_date - pr_created_date))/60)) AS NUMERIC) / NULLIF(1440, 0)) FROM pr_comment_list WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Time to Initial PR Review [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 14, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT AVG(CAST((EXTRACT(EPOCH FROM (closed_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_close FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND status IN ('CLOSED', 'MERGED') AND ('${repo_id:csv}' = '' OR pr.base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Resolution Time [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 18, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(100 * SUM(CASE WHEN CAST((EXTRACT(EPOCH FROM (closed_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0) < $prrt_sla THEN 1 ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(*), 0) FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND status IN ('CLOSED', 'MERGED') AND ('${repo_id:csv}' = '' OR pr.base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Resolution Rate within SLA [Last Month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 20, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(100 * COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' - INTERVAL '1 MONTH' AND CURRENT_DATE - INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR pr.base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Closed W/O Merging Ratio [Last Month]", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 22, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "OSS Maintainer Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '-' || id FROM repos", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '-' || id FROM repos", + "refresh": 1, + "regex": "/^(?.*)-(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "24", + "value": "24" + }, + "hide": 0, + "label": "Time to Initial Issue Response SLA (in hours)", + "name": "iir_sla", + "options": [ + { + "selected": true, + "text": "24", + "value": "24" + } + ], + "query": "24", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": true, + "text": "7", + "value": "7" + }, + "hide": 0, + "label": "PR Resolution Time SLA (in days)", + "name": "prrt_sla", + "options": [ + { + "selected": true, + "text": "7", + "value": "7" + } + ], + "query": "7", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "good first issue", + "value": "good first issue" + }, + "hide": 0, + "label": "Label for good first issues", + "name": "label_gfi", + "options": [ + { + "selected": true, + "text": "good first issue", + "value": "good first issue" + } + ], + "query": "good first issue", + "skipUrlSync": false, + "type": "textbox" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Contributor Experience", + "uid": "bwsP5Nz4z-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-average-requirement-lead-time-by-assignee.json b/grafana/dashboards/postgresql/demo-average-requirement-lead-time-by-assignee.json new file mode 100644 index 00000000000..023e8cf18e6 --- /dev/null +++ b/grafana/dashboards/postgresql/demo-average-requirement-lead-time-by-assignee.json @@ -0,0 +1,250 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 1, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Back to previous page", + "tooltip": "", + "type": "link", + "url": "/grafana/d/SupYz7c7z/demo-how-fast-do-we-respond-to-customer-requirements?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Compare the average time spent in each state of a requirement in the last 2 months.\n2. The requirements being calculated are the requirements delivered in the last 2 months.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Days", + "axisPlacement": "auto", + "axisSoftMax": 8, + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 98, + "options": { + "barWidth": 0.6, + "groupWidth": 0.5, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "vertical", + "showValue": "always", + "text": { + "valueSize": 16 + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(resolution_date AS DATE)) + 1) AS time, assignee_name AS assignee, CAST(AVG(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS lead_time FROM issues AS i WHERE type = 'REQUIREMENT' AND assignee_id <> '' AND $__timeFilter(resolution_date) GROUP BY 1, 2), this_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) AS this_month), last_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-1 MONTH' AS last_month), the_month_before_last AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-2 MONTH' AS the_month_before_last), this_month_record AS (SELECT assignee, lead_time AS this_month_count, time AS this_month FROM _requirements WHERE time IN (SELECT this_month FROM this_month)), last_month_record AS (SELECT assignee, lead_time AS last_month_count, time AS last_month FROM _requirements WHERE time IN (SELECT last_month FROM last_month)), the_month_before_last_record AS (SELECT assignee, lead_time AS the_month_before_last_count, time AS the_month_before_last FROM _requirements WHERE time IN (SELECT the_month_before_last FROM the_month_before_last)) SELECT COALESCE(NULLIF(tmr.assignee, ''), NULLIF(lmr.assignee, ''), tmblr.assignee) AS \"Assignee\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\", CASE WHEN lmr.last_month_count IS NULL OR tmr.this_month_count IS NULL THEN '-' ELSE ROUND(CAST(100 * (tmr.this_month_count - lmr.last_month_count) AS NUMERIC) / NULLIF(lmr.last_month_count, 0), 1)::TEXT || '%' END AS \"Changes in last 2 month\" FROM the_month_before_last_record AS tmblr LEFT JOIN last_month_record AS lmr ON tmblr.assignee = lmr.assignee LEFT JOIN this_month_record AS tmr ON tmblr.assignee = tmr.assignee ORDER BY 2 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Requirement Lead Time by Assignee (day)", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 100, + "options": { + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(resolution_date AS DATE)) + 1) AS time, assignee_name AS assignee, CAST(AVG(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS lead_time FROM issues AS i WHERE type = 'REQUIREMENT' AND assignee_id <> '' AND $__timeFilter(resolution_date) GROUP BY 1, 2), this_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) AS this_month), last_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-1 MONTH' AS last_month), the_month_before_last AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-2 MONTH' AS the_month_before_last), this_month_record AS (SELECT assignee, lead_time AS this_month_count, time AS this_month FROM _requirements WHERE time IN (SELECT this_month FROM this_month)), last_month_record AS (SELECT assignee, lead_time AS last_month_count, time AS last_month FROM _requirements WHERE time IN (SELECT last_month FROM last_month)), the_month_before_last_record AS (SELECT assignee, lead_time AS the_month_before_last_count, time AS the_month_before_last FROM _requirements WHERE time IN (SELECT the_month_before_last FROM the_month_before_last)) SELECT COALESCE(NULLIF(tmr.assignee, ''), NULLIF(lmr.assignee, ''), tmblr.assignee) AS \"Assignee\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\", CASE WHEN lmr.last_month_count IS NULL OR tmr.this_month_count IS NULL THEN '-' ELSE ROUND(CAST(100 * (tmr.this_month_count - lmr.last_month_count) AS NUMERIC) / NULLIF(lmr.last_month_count, 0), 1)::TEXT || '%' END AS \"Changes in last 2 month\" FROM the_month_before_last_record AS tmblr LEFT JOIN last_month_record AS lmr ON tmblr.assignee = lmr.assignee LEFT JOIN this_month_record AS tmr ON tmblr.assignee = tmr.assignee ORDER BY 2 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "type": "table" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 102, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Average Requirement Lead Time By Assignee", + "uid": "q27fk7cnk-pg", + "version": 6 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-commit-count-by-author.json b/grafana/dashboards/postgresql/demo-commit-count-by-author.json new file mode 100644 index 00000000000..039a7ee3c2b --- /dev/null +++ b/grafana/dashboards/postgresql/demo-commit-count-by-author.json @@ -0,0 +1,257 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Back to previous page", + "tooltip": "", + "type": "link", + "url": "/grafana/d/ddREk75nk/demo-is-this-month-more-productive-than-last?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "The number of commits from different commit authors.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 50, + "options": { + "barWidth": 0.6, + "groupWidth": 0.5, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "text": { + "valueSize": 14 + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH commit_data AS (SELECT CAST(authored_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(authored_date AS DATE)) + 1) AS time, c.author_name, COUNT(*) AS commit_count FROM commits AS c WHERE c.message::text NOT LIKE '%Merge%' AND $__timeFilter(authored_date) GROUP BY 2, 1 ORDER BY 2 NULLS FIRST, 1 NULLS FIRST), this_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) AS this_month), last_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-1 MONTH' AS last_month), the_month_before_last AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-2 MONTH' AS the_month_before_last), this_month_record AS (SELECT author_name, commit_count AS this_month_count, time AS this_month FROM commit_data WHERE time IN (SELECT this_month FROM this_month)), last_month_record AS (SELECT author_name, commit_count AS last_month_count, time AS last_month FROM commit_data WHERE time IN (SELECT last_month FROM last_month)), the_month_before_last_record AS (SELECT author_name, commit_count AS the_month_before_last_count, time AS the_month_before_last FROM commit_data WHERE time IN (SELECT the_month_before_last FROM the_month_before_last)) SELECT COALESCE(NULLIF(tmr.author_name, ''), NULLIF(lmr.author_name, ''), tmblr.author_name) AS \"Author Name\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\" FROM this_month_record AS tmr RIGHT JOIN last_month_record AS lmr ON tmr.author_name = lmr.author_name RIGHT JOIN the_month_before_last_record AS tmblr ON lmr.author_name = tmblr.author_name UNION SELECT COALESCE(NULLIF(tmr.author_name, ''), NULLIF(lmr.author_name, ''), tmblr.author_name) AS \"Author Name\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\" FROM this_month_record AS tmr LEFT JOIN last_month_record AS lmr ON tmr.author_name = lmr.author_name LEFT JOIN the_month_before_last_record AS tmblr ON lmr.author_name = tmblr.author_name ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Commit Count by Author [Top 20]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 38, + "options": { + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH commit_data AS (SELECT CAST(authored_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(authored_date AS DATE)) + 1) AS time, c.author_name, COUNT(*) AS commit_count FROM commits AS c WHERE c.message::text NOT LIKE '%Merge%' AND $__timeFilter(authored_date) GROUP BY 2, 1 ORDER BY 2 NULLS FIRST, 1 NULLS FIRST), this_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) AS this_month), last_month AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-1 MONTH' AS last_month), the_month_before_last AS (SELECT CAST(CURRENT_DATE AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(CURRENT_DATE AS DATE)) + 1) + INTERVAL '-2 MONTH' AS the_month_before_last), this_month_record AS (SELECT author_name, commit_count AS this_month_count, time AS this_month FROM commit_data WHERE time IN (SELECT this_month FROM this_month)), last_month_record AS (SELECT author_name, commit_count AS last_month_count, time AS last_month FROM commit_data WHERE time IN (SELECT last_month FROM last_month)), the_month_before_last_record AS (SELECT author_name, commit_count AS the_month_before_last_count, time AS the_month_before_last FROM commit_data WHERE time IN (SELECT the_month_before_last FROM the_month_before_last)) SELECT COALESCE(NULLIF(tmr.author_name, ''), NULLIF(lmr.author_name, ''), tmblr.author_name) AS \"Author Name\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\", CASE WHEN lmr.last_month_count IS NULL OR tmr.this_month_count IS NULL THEN '-' ELSE ROUND(CAST(100 * (tmr.this_month_count - lmr.last_month_count) AS NUMERIC) / NULLIF(lmr.last_month_count, 0), 1)::TEXT || '%' END AS \"Changes in last 2 month\" FROM this_month_record AS tmr RIGHT JOIN last_month_record AS lmr ON tmr.author_name = lmr.author_name RIGHT JOIN the_month_before_last_record AS tmblr ON lmr.author_name = tmblr.author_name UNION SELECT COALESCE(NULLIF(tmr.author_name, ''), NULLIF(lmr.author_name, ''), tmblr.author_name) AS \"Author Name\", COALESCE(tmblr.the_month_before_last_count, 0) AS \"The Month before Last\", COALESCE(lmr.last_month_count, 0) AS \"Last Month\", COALESCE(tmr.this_month_count, 0) AS \"This Month\", CASE WHEN lmr.last_month_count IS NULL OR tmr.this_month_count IS NULL THEN '-' ELSE ROUND(CAST(100 * (tmr.this_month_count - lmr.last_month_count) AS NUMERIC) / NULLIF(lmr.last_month_count, 0), 1)::TEXT || '%' END AS \"Changes in last 2 month\" FROM this_month_record AS tmr LEFT JOIN last_month_record AS lmr ON tmr.author_name = lmr.author_name LEFT JOIN the_month_before_last_record AS tmblr ON lmr.author_name = tmblr.author_name ORDER BY 4 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "type": "table" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 52, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Commit Count by Author", + "uid": "F0iYknc7z-pg", + "version": 3 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-detailed-bug-info.json b/grafana/dashboards/postgresql/demo-detailed-bug-info.json new file mode 100644 index 00000000000..3529a0bbe81 --- /dev/null +++ b/grafana/dashboards/postgresql/demo-detailed-bug-info.json @@ -0,0 +1,315 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 2, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Back to previous page", + "tooltip": "", + "type": "link", + "url": "/grafana/d/G4DEk75nz/demo-was-our-quality-improved-or-not?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 32, + "options": { + "barWidth": 0.3, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "text": { + "valueSize": 16 + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH bugs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COUNT(*) AS bug_count FROM issues AS i WHERE type = 'BUG' AND $__timeFilter(created_date) GROUP BY 1 ORDER BY 1 DESC NULLS LAST) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, bug_count AS \"Bug Count over Month\" FROM bugs ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Bug count over Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 405 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "summary" + }, + "properties": [ + { + "id": "custom.width", + "value": 538 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "description" + }, + "properties": [ + { + "id": "custom.width", + "value": 753 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "created_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.width", + "value": 284 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 30, + "options": { + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT title, description, CASE WHEN assignee_id = '' THEN '-' ELSE assignee_name END AS assignee, status, created_date, url FROM issues AS i WHERE type = 'BUG' AND $__timeFilter(created_date) ORDER BY created_date DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "type": "table" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 34, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Detailed Bug Info", + "uid": "s48Lzn5nz-pg", + "version": 8 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-homepage.json b/grafana/dashboards/postgresql/demo-homepage.json new file mode 100644 index 00000000000..2100d25700f --- /dev/null +++ b/grafana/dashboards/postgresql/demo-homepage.json @@ -0,0 +1,191 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 10, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "datasource": null, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": null, + "links": [], + "options": { + "content": "\n
\n

How fast do we respond to customer requirements?

\n
\n
", + "mode": "html" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT create_time AS \"time\", progress FROM ca_analysis WHERE $__timeFilter(create_time) ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "type": "text" + }, + { + "cacheTimeout": null, + "datasource": null, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 3, + "interval": null, + "links": [], + "options": { + "content": "\n
\n

Was our quality improved or not?

\n
\n
", + "mode": "html" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT create_time AS \"time\", progress FROM ca_analysis WHERE $__timeFilter(create_time) ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "type": "text" + }, + { + "cacheTimeout": null, + "datasource": null, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [], + "options": { + "content": "\n
\n

Is this month more productive than last?

\n
\n
", + "mode": "html" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": false, + "rawSql": "SELECT create_time AS \"time\", progress FROM ca_analysis WHERE $__timeFilter(create_time) ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Homepage", + "uid": "0Rjxknc7z-pg", + "version": 2 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-how-fast-do-we-respond-to-customer-requirements.json b/grafana/dashboards/postgresql/demo-how-fast-do-we-respond-to-customer-requirements.json new file mode 100644 index 00000000000..cf477e80622 --- /dev/null +++ b/grafana/dashboards/postgresql/demo-how-fast-do-we-respond-to-customer-requirements.json @@ -0,0 +1,223 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 11, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 17, + "interval": "", + "links": [ + { + "title": "Drill down by Assignee", + "url": "/grafana/d/q27fk7cnk/demo-average-requirement-lead-time-by-assignee?orgId=1" + } + ], + "options": { + "barWidth": 0.25, + "groupWidth": 0.74, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "text": { + "valueSize": 18 + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(resolution_date AS DATE)) + 1) AS time, CAST(AVG(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS lead_time_days FROM issues AS i WHERE type = 'REQUIREMENT' AND $__timeFilter(resolution_date) GROUP BY time) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, lead_time_days AS \"Average Requirement Lead Time (day)\" FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average Requirement Lead Time over time (day)", + "type": "barchart" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 5, + "x": 0, + "y": 9 + }, + "id": 103, + "options": { + "content": "
\n\n[Drill down by assignee](/d/q27fk7cnk/demo-average-requirement-lead-time-by-assignee?orgId=1)\n ", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 101, + "options": { + "content": "
\n \"Merico\"\n

MARI Guide - Requirement Lead Time

\n
\n\nSection | Description\n:----------------- | :-------------\nMetric Definition | Total duration of requirements from proposal to delivery. It can be divided by flow status in the practice domain or project management system to count the time share of each phase and help locate the links that drag out the requirement delivery cycle.\nMetric Value | The Requirement Lead Time reflects the rapid responsiveness of the R&D team.
In theory, the faster you can deliver value to customers, the better, but other aspects such as whether the delivered value meets customer expectations, requirement throughput, and delivery quality must be considered together. Fast delivery does not necessarily equate to good R&D practices.\n\n***\n#### *M (Measure)*\n1. Count the average or 80th percentile requirement lead time for different times.\n2. Counts the average or 80th percentile requirement lead time for different projects.
\n3. Count the length of time that requirements stay in different practice domains (requirements analysis, design, development, testing, release) or in different states.\n\n##### *A (Analyze)*\n1. Compare the requirement delivery speed of different projects to find the fastest and slowest delivering projects.\n2. Analyze the trend of the average requirement lead time within each cycle, make a vertical comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.\n3. Analyze the trend of the delivery cycle of 80% of the requirements within each cycle, make a longitudinal comparison, and locate the key points such as maximum value, minimum value, continuous up cycle, and continuous down cycle.

\nWhy choose the 80% quantile instead of using the average?
\nThe point of statistics is to make predictions with real and valid data to support better decisions, while the mean and median cannot have the role of supporting predictions.
\nTypically, the mean and 80% quantile statistics will appear twice as far apart, and the 80% and 99% quantile tend to be approximately twice as related.
\nTherefore, the 80% quantile is a good balance point for prediction.\n
\n4. Analysis compares the length of time requirement stays in different practice domains or different states to identify the most time-consuming links and find the key bottlenecks that affect overall delivery speed.\n5. Requirement lead time is correlated with requirement throughput to identify whether the requirement delivery trend is healthy or not.\n - Healthy trend: requirement lead time is shortened and requirement throughput is increased.\n - Unhealthy trend: longer requirement lead time and lower requirement throughput.\n\n\n##### *R (Review)*\nBased on the analysis results, focus on a few key points and use Ishikawa diagram (fishbone diagram) or Root Cause Analysis (RCA) to conduct root cause analysis, research and review. For example, if the requirement delivery cycle becomes longer in several consecutive statistical cycles, it is necessary to further investigate the length of stay of requirements in different phases and find the longest phase for root cause analysis.\n\n1. The requirements phase takes too long: unclear requirements, frequent changes, overly granular requirements, requirements priorities not clearly defined, insufficient resources or experience of requirements analysts or product managers?\n2. The design phase takes too long: unclear requirement documents, insufficient resources or experience of R&D leaders or architects?\n3. The development phase takes too long: unclear design documents, uneven task distribution, high stream load (parallel tasks), too much technical debt, too many bugs, insufficient resources or experience of developers?\n4. The testing phase takes too long: unclear requirements documentation, poor code quality, few automated tests, insufficient resources or experience of testers?\n5. The release phase takes too long: too long build or deployment time, insufficient resources or experience of operation and maintenance staff?\n\n##### *I (Improve)*\nBased on the review results, focus on the key root causes, and give targeted improvement measures in terms of norms, processes, tools and behaviors, etc., with clear improvement targets, improvement measures, verification cycles and responsible persons.\n\nThe following are the improvement ideas for reference:\n\n1. Communicate with customers or business parties to clarify requirements, reasonably disassemble requirements and define priorities, and invite business parties, R&D leaders and testing leaders to review requirements.\n2. Invite product managers, R&D personnel, and test leaders to conduct design reviews.\n3. Reduce requirement or task granularity, distribute tasks evenly, reduce flow load, increase unit testing, and solve technical debt and code problems in time to reduce the number of bugs and rework.\n4. Test left, develop self-test, code review, increase automated testing, and improve continuous integration capabilities.\n5. Automate deployment, shorten build time, and improve continuous delivery.\n6. Reasonable resource allocation and necessary training for each job holder.\n7. The improvement results should also be quantifiable to facilitate continuous metrics and tracking of improvement effects.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 105, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-How fast do we respond to customer requirements?", + "uid": "SupYz7c7z-pg", + "version": 4 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-is-this-month-more-productive-than-last.json b/grafana/dashboards/postgresql/demo-is-this-month-more-productive-than-last.json new file mode 100644 index 00000000000..f6a4dcd75ae --- /dev/null +++ b/grafana/dashboards/postgresql/demo-is-this-month-more-productive-than-last.json @@ -0,0 +1,253 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 12, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "links": [ + { + "targetBlank": true, + "title": "Break down by user1", + "url": "/grafana/d/cHnTA7K7z/is-this-month-more-productive-than-last-by-users?orgId=1" + } + ], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Commit Count" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 36, + "links": [ + { + "targetBlank": false, + "title": "Drill down by Commit Author", + "url": "/d/F0iYknc7z/demo-commit-count-by-author?orgId=1" + } + ], + "options": { + "barWidth": 0.25, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "text": {}, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _commits AS (SELECT CAST(authored_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(authored_date AS DATE)) + 1) AS time, COUNT(*) AS commit_count FROM commits WHERE message::text NOT LIKE '%Merge%' AND $__timeFilter(authored_date) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, commit_count AS \"Commit Count\" FROM _commits ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Commit Count over Time", + "type": "barchart" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 5, + "x": 0, + "y": 9 + }, + "id": 40, + "options": { + "content": "
\n\n[Drill down by commit author](/d/F0iYknc7z/demo-commit-count-by-author?orgId=1)\n ", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 38, + "options": { + "content": "
\n \"Merico\"\n

MARI Guide - Commit Count

\n
\n\nSection | Description\n:------------------ | :-------------\nMetric Definition | The total number of commits.\nMetric Value | The number of commits reflects the frequency of code commits and encourages multiple small step commits per day. While paying attention to the number of times, you should also observe the number of lines of code per commit.\n\n***\n\n#### *M (Measure)*\nThe number of commits can be counted in time, project, team or individual dimensions, and the above 4 dimensions can be cross-tabulated, e.g., the number of commits per week for a project, the average number of commits per person per day for a team.\n\n##### *A (Analyze)*\n1. By project dimension, analyze and compare the number of code commits of different projects horizontally, and locate the projects with the maximum, minimum and lower than average values for research.\n2. By team dimension, analyze and compare the number of code commits of different teams horizontally, and locate the teams with the maximum, minimum, and lower than average values for research.\n3. By individual dimension, analyze and compare the number of code commits of different personnel horizontally, and locate the individuals with the maximum, minimum, and lower than average values for research.\n4. Analyze the trend of the number of code commits in each cycle, compare vertically, and locate the maximum, minimum, and continuously increasing or decreasing cycles for research.\n\n##### *R (Review)*\nBased on the results of metrics analysis, use Ishikawa diagram (fishbone diagram) or root cause analysis (RCA) to analyze, investigate and review the root causes of projects/teams/individuals with low (e.g. lower than average) code submission times in a cycle:\n1. too many complicated requirements with large granularity (estimated workload)?\n2. uneven and full task distribution?\n3. no specification for small step submission?\n\n##### *I (Improve)*\nBased on the review results, focus on the key root causes, give targeted improvement measures in terms of specification, process, tools, behavior, etc., and clarify the improvement target, improvement measures, verification cycle and responsible person.\nTo address the root causes of low code submission in the cycle, the following improvement measures were taken: 1.\n1. reduce the granularity of requirements.\n2. distribute tasks evenly so that each person has a full workload each day.\n3. set small step commit specifications, such as an average number of commits per person per day greater than 1, an average number of lines per commit less than 100, etc.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 42, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Is this month more productive than last?", + "uid": "ddREk75nk-pg", + "version": 3 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/demo-was-our-quality-improved-or-not.json b/grafana/dashboards/postgresql/demo-was-our-quality-improved-or-not.json new file mode 100644 index 00000000000..4735a4f17a8 --- /dev/null +++ b/grafana/dashboards/postgresql/demo-was-our-quality-improved-or-not.json @@ -0,0 +1,220 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 6, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": false, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/0Rjxknc7z/demo-homepage?orgId=1" + } + ], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "links": [ + { + "targetBlank": false, + "title": "See Detailed Bug Info", + "url": "/grafana/d/s48Lzn5nz/demo-detailed-bug-info?orgId=1" + } + ], + "options": { + "barWidth": 0.3, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "always", + "text": { + "valueSize": 14 + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH line_of_code AS (SELECT CAST(authored_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(authored_date AS DATE)) + 1) AS time, SUM(additions + deletions) AS line_count FROM commits WHERE message::text NOT LIKE 'Merge%' AND $__timeFilter(authored_date) GROUP BY 1), bug_count AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COUNT(*) AS bug_count FROM issues AS i WHERE type = 'BUG' AND $__timeFilter(created_date) GROUP BY 1), bug_count_per_1k_loc AS (SELECT loc.time, CAST(1.0 * bc.bug_count AS NUMERIC) / NULLIF(loc.line_count, 0) * 1000 AS bug_count_per_1k_loc FROM line_of_code AS loc LEFT JOIN bug_count AS bc ON bc.time = loc.time WHERE NOT bc.bug_count IS NULL AND NOT loc.line_count IS NULL AND loc.line_count <> '0') SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, bug_count_per_1k_loc AS \"Bug Count per 1000 Lines of Code\" FROM bug_count_per_1k_loc ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Bug Count per 1k Lines of Code", + "type": "barchart" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 5, + "x": 0, + "y": 8 + }, + "id": 37, + "options": { + "content": "
\n\n[See Detailed Bug Info](/d/s48Lzn5nz/demo-detailed-bug-info?orgId=1)\n ", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 32, + "options": { + "content": "
\n \"Merico\"\n

MARI Guide - Bug Count per 1k Lines of Code

\n
\n\nSection | Description\n:----------------- | :-------------\nMetric Definition | The ratio of the number of bugs found to the corresponding amount of code or code changes, to characterize the density of overall bugs, including bugs found in testing and online. For example, bug count per 1k lines of code, bug count per 1k code equivalent.\nMetric Value | The bug rate, as a quality indicator, represents the density of bugs, and is one of the important indicators used to assess the quality of software products and testing quality. Usually, the cost of fixing detected bugs is higher in the later stage of the software development life cycle, and this metric is valuable for analyzing and evaluating both online quality and bug fixing cost.\n\n***\n#### *M (Measure)*\n1. Bug count per 1k lines of code by project.\n2. Trends in 'Bug count per 1k lines of code' over time.\n3. Measure historical data to establish year-over-year and historical baseline reference values for 'Bug count per 1k lines of code'.\n\n##### *A (Analyze)*\n1. Year-over-year analysis: The bug rate of similar projects in the same period is compared and analyzed, and the improvement effect of product quality is observed through the rise and fall of the data.\n2. Circumferential analysis: Analyze the bug rate of projects in the recent year, analyze the change of online bug rate according to the time axis, and compare with the historical baseline at the same time to give a judgment analysis of the rise and fall of indicators.\n3. Trend analysis: analyze the trend of bug rate over time (days, weeks, months), judge the trend rise, and evaluate whether the stable cycle of product quality is reasonable by observing changes such as trend slowing-down and smoothing.\n4. Horizontal analysis: Compare the bug rate of multiple projects as a reference to evaluate the quality of software products online.\n5. Classification analysis: Classify and analyze the types of bugs, severity levels, and modules they belong to, and identify the key issues that show aggregated distribution.\n\n##### *R (Review)*\nFor the high severity level of online bugs should be a complete review, according to the timeline, role dimensions, the sequence of events on the root cause of bugs to dig, locate the key issues.\nAccording to the quantitative conclusions drawn from the analysis, further data drilling and root cause mining can be organized for bugs in several dimensions, including whether they are missed, the module they belong to, the cause, the occurrence cycle, and the resolution.\n1. bug escape rate: derived from [number of online bugs / (number of online bugs + number of bugs found in test)], this indicator can be compared with historical data, if the data exceeds the acceptable range of history and testing department, then it is necessary to conduct leak analysis. If the data exceeds the acceptable interval of history and testing department, it is necessary to analyze the missed test and confirm whether the use case is missed or not covered, so as to strengthen the use case design and management.\n2. defective module: The defective module can be located to the key module where the problem is concentrated, and targeted improvement measures are required for each link from requirements, design, development to testing, and typical problems are located for the bugive module to establish targeted measures.\n3. bug generation causes: through the cause analysis of bugs, similar bugs bugs can be put together, so that bugs belonging to the same category and accounting for a high percentage of bugs will be highlighted, so that it is easy to take out the bugs with more concentrated causes and jointly discuss the next improvement measures to precisely reduce the number of similar bugs.\n4. bug occurrence cycle: analyze the bug occurrence cycle, determine whether the users use the system frequently, whether the system has been updated or optimized, whether the system has been refactored, etc., which will cause a long and short bug occurrence cycle after the launch, through the analysis of the bug cycle length, to draw some valuable conclusions about the stability of the system.\n5. bug resolution: statistics on the resolution of bugs, which bugs are not reproduced, which are temporarily handled, and which are in need of continuous improvement. For the temporary resolution of bugs, analyze whether it will cause another bug elsewhere in the system, whether the user can receive the temporary resolution, what development and testing need to focus on similar bugs, and whether similar bugs need to be tested elsewhere for horizontal expansion. Those that require continuous improvement need to be further tracked by testing until the problem is resolved.\n\n##### *I (Improve)*\nThrough root cause mining, starting from the key bugs, the key problems of each link to locate, in accordance with the principle that the later the bugs are found, the higher the cost and complexity of the solution, in addition to the test design and implementation to start improving (refer to the improvement links of the number of bugs on the line), more should start the construction of quality from the upstream of the software engineering stage, to achieve the forward movement of the bug discovery stage, such as\n1. optimizing static scan rules according to the type, number and severity level of static scan problems to reduce false positives and expose as many serious problems as possible (quality over quantity).\n2. Define the requirement of resolution ratio for different severity level problems to control the backlog of serious problems.\n3. Establish code review system, strategy, and encourage the promotion of code review implementation.\n4. Establish unit test coverage ratio or unit test coverage condition requirements, e.g. functions with circle complexity greater than 10 shall be covered by unit tests.\nImplement improvement measures and clarify the improvement target, improvement measures, verification cycle and responsible person. Do a new round of MARI (Measure, Analysis, Review, and Verification) for the improvement effect to quantify the improvement effect.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 39, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 30, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Demo-Was our quality improved or not?", + "uid": "G4DEk75nz-pg", + "version": 4 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/developer-productivity-hours.json b/grafana/dashboards/postgresql/developer-productivity-hours.json new file mode 100644 index 00000000000..ba9848d1c43 --- /dev/null +++ b/grafana/dashboards/postgresql/developer-productivity-hours.json @@ -0,0 +1,267 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Chat and completion events by hour of day", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(hr AS TEXT), 2, '0') AS \"Hour\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hr, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hr, 0 AS chat, COUNT(*) AS comp FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS t GROUP BY hr ORDER BY hr NULLS FIRST", + "refId": "A" + } + ], + "title": "AI Activity by Hour of Day (UTC)", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average prompt complexity and response richness by hour", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", ROUND(AVG(prompt_length)) AS \"Avg Prompt Length\", ROUND(AVG(response_length)) AS \"Avg Response Length\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", + "refId": "A" + } + ], + "title": "Prompt & Response Length by Hour", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Steering and spec mode usage concentration by hour", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(EXTRACT(HOUR FROM timestamp) AS TEXT), 2, '0') AS \"Hour\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) ORDER BY EXTRACT(HOUR FROM timestamp) NULLS FIRST", + "refId": "A" + } + ], + "title": "Feature Usage by Hour", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Day-of-week activity pattern", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) WHEN 1 THEN 'Sun' WHEN 2 THEN 'Mon' WHEN 3 THEN 'Tue' WHEN 4 THEN 'Wed' WHEN 5 THEN 'Thu' WHEN 6 THEN 'Fri' WHEN 7 THEN 'Sat' END AS \"Day\", SUM(chat) AS \"Chat Events\", SUM(comp) AS \"Completions\" FROM (SELECT timestamp, COUNT(*) AS chat, 0 AS comp FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY timestamp UNION ALL SELECT timestamp, 0, COUNT(*) FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY timestamp) AS t GROUP BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) ORDER BY EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) NULLS FIRST", + "refId": "A" + } + ], + "title": "AI Activity by Day of Week", + "type": "barchart" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "productivity", + "hours" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Developer AI Productivity Hours", + "uid": "kiro_productivity_hours-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-by-team.json b/grafana/dashboards/postgresql/dora-by-team.json new file mode 100644 index 00000000000..b8f227c64ac --- /dev/null +++ b/grafana/dashboards/postgresql/dora-by-team.json @@ -0,0 +1,1254 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 43, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 16, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See [how to config](https://devlake.apache.org/docs/DORA) this dashboard\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- You can validate/debug this dashboard with the [DORA validation dashboard](/grafana/d/KGkUnV-Vz/dora-dashboard-validation) \n- You also need to do [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard. \n- DORA benchmarks vary in different years. You can switch the benchmarks to change them.\n- In DORA's official report in 2023, metric 'failed deployment recovery time' has replaced 'MTTR'.\n- How does this work? \n - Gets the author of the specific commit and then navigates to the team the user belongs to. \n - Gets the team from the PR's author. \n - Gets the team from the commit author.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "low" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "medium" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "high" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "elite" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 8, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE 0 END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5), _metric_deployment_frequency AS (SELECT 'Deployment frequency' AS metric, CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN 'On-demand(elite)' WHEN median_number_of_deployment_days_per_week >= 1 THEN 'Between once per day and once per week(high)' WHEN median_number_of_deployment_days_per_month >= 1 THEN 'Between once per week and once per month(medium)' WHEN median_number_of_deployment_days_per_month < 1 AND NOT is_collected IS NULL THEN 'Fewer than once per month(low)' ELSE 'N/A. Please check if you have collected deployments.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN 'On-demand(elite)' WHEN median_number_of_deployment_days_per_month >= 1 THEN 'Between once per day and once per month(high)' WHEN median_number_of_deployment_days_per_six_months >= 1 THEN 'Between once per month and once every 6 months(medium)' WHEN median_number_of_deployment_days_per_six_months < 1 AND NOT is_collected IS NULL THEN 'Fewer than once per six months(low)' ELSE 'N/A. Please check if you have collected deployments.' END ELSE 'Invalid dora report' END AS value FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months), _pr_stats /* Metric 2: median lead time for changes */ AS (/* get the cycle time of PRs deployed by the deployments finished in the selected period */ SELECT DISTINCT pr.id, ppm.pr_cycle_time FROM pull_requests AS pr JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _median_change_lead_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats), _median_change_lead_time AS (/* use median PR cycle time as the median change lead time */ SELECT MAX(pr_cycle_time) AS median_change_lead_time FROM _median_change_lead_time_ranks WHERE ranks <= 0.5), _metric_change_lead_time AS (SELECT 'Lead time for changes' AS metric, CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_change_lead_time < 24 * 60 THEN 'Less than one day(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN 'Between one day and one week(high)' WHEN median_change_lead_time < 30 * 24 * 60 THEN 'Between one week and one month(medium)' WHEN median_change_lead_time >= 30 * 24 * 60 THEN 'More than one month(low)' ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_change_lead_time < 60 THEN 'Less than one hour(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN 'Less than one week(high)' WHEN median_change_lead_time < 180 * 24 * 60 THEN 'Between one week and six months(medium)' WHEN median_change_lead_time >= 180 * 24 * 60 THEN 'More than six months(low)' ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END ELSE 'Invalid dora report' END AS value FROM _median_change_lead_time), _deployments /* Metric 3: change failure rate */ AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate AS (SELECT CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments), _is_collected_data AS (SELECT CASE WHEN COUNT(i.id) = 0 AND COUNT(cdc.id) = 0 THEN 'No All' WHEN COUNT(i.id) = 0 THEN 'No Incidents' WHEN COUNT(cdc.id) = 0 THEN 'No Deployments' END AS is_collected FROM (SELECT 1) AS dummy LEFT JOIN incidents AS i ON 1 = 1 LEFT JOIN cicd_deployment_commits AS cdc ON 1 = 1), _metric_cfr AS (SELECT 'Change failure rate' AS metric, CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.05 THEN '0-5%(elite)' WHEN change_failure_rate <= 0.10 THEN '5%-10%(high)' WHEN change_failure_rate <= 0.15 THEN '10%-15%(medium)' WHEN change_failure_rate > 0.15 THEN '> 15%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.15 THEN '0-15%(elite)' WHEN change_failure_rate <= 0.20 THEN '16%-20%(high)' WHEN change_failure_rate <= 0.30 THEN '21%-30%(medium)' WHEN change_failure_rate > 0.30 THEN '> 30%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END ELSE 'Invalid dora report' END AS value FROM _change_failure_rate, _is_collected_data), _incidents_for_deployments /* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5), _metric_recovery_time_2023_report AS (SELECT 'Failed deployment recovery time' AS metric, CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_recovery_time < 60 THEN 'Less than one hour(elite)' WHEN median_recovery_time < 24 * 60 THEN 'Less than one day(high)' WHEN median_recovery_time < 7 * 24 * 60 THEN 'Between one day and one week(medium)' WHEN median_recovery_time >= 7 * 24 * 60 THEN 'More than one week(low)' ELSE 'N/A. Please check if you have collected deployments or incidents.' END END AS median_recovery_time FROM _median_recovery_time), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND $__timeFilter(i.resolution_date)), _median_mttr_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _median_mttr AS (SELECT MAX(lead_time_minutes) AS median_time_to_resolve FROM _median_mttr_ranks WHERE ranks <= 0.5), _metric_mttr_2021_report AS (SELECT 'Time to restore service' AS metric, CASE WHEN ('$dora_report') = '2021' THEN CASE WHEN median_time_to_resolve < 60 THEN 'Less than one hour(elite)' WHEN median_time_to_resolve < 24 * 60 THEN 'Less than one day(high)' WHEN median_time_to_resolve < 7 * 24 * 60 THEN 'Between one day and one week(medium)' WHEN median_time_to_resolve >= 7 * 24 * 60 THEN 'More than one week(low)' ELSE 'N/A. Please check if you have collected incidents.' END END AS median_time_to_resolve FROM _median_mttr), _metric_mrt_or_mm AS (SELECT metric, median_recovery_time AS value FROM _metric_recovery_time_2023_report WHERE ('$dora_report') = '2023' UNION SELECT metric, median_time_to_resolve AS value FROM _metric_mttr_2021_report WHERE ('$dora_report') = '2021'), _final_results AS (SELECT DISTINCT db.id, db.metric, db.low, db.medium, db.high, db.elite, m1.metric AS _metric, m1.value FROM dora_benchmarks AS db LEFT JOIN _metric_deployment_frequency AS m1 ON db.metric = m1.metric WHERE NOT m1.metric IS NULL AND db.dora_report = ('$dora_report') UNION SELECT DISTINCT db.id, db.metric, db.low, db.medium, db.high, db.elite, m2.metric AS _metric, m2.value FROM dora_benchmarks AS db LEFT JOIN _metric_change_lead_time AS m2 ON db.metric = m2.metric WHERE NOT m2.metric IS NULL AND db.dora_report = ('$dora_report') UNION SELECT DISTINCT db.id, db.metric, db.low, db.medium, db.high, db.elite, m3.metric AS _metric, m3.value FROM dora_benchmarks AS db LEFT JOIN _metric_cfr AS m3 ON db.metric = m3.metric WHERE NOT m3.metric IS NULL AND db.dora_report = ('$dora_report') UNION SELECT DISTINCT db.id, db.metric, db.low, db.medium, db.high, db.elite, m4.metric AS _metric, m4.value FROM dora_benchmarks AS db LEFT JOIN _metric_mrt_or_mm AS m4 ON db.metric = m4.metric WHERE NOT m4.metric IS NULL AND db.dora_report = ('$dora_report')) SELECT metric, CASE WHEN low = value THEN low ELSE NULL END AS low, CASE WHEN medium = value THEN medium ELSE NULL END AS medium, CASE WHEN high = value THEN high ELSE NULL END AS high, CASE WHEN elite = value THEN elite ELSE NULL END AS elite FROM _final_results ORDER BY id NULLS FIRST", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Overall DORA Metrics", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Deployment Frequency$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE 0 END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN median_number_of_deployment_days_per_week || ' deployment days per week(elite)' WHEN median_number_of_deployment_days_per_week >= 1 THEN median_number_of_deployment_days_per_week || ' deployment days per week(high)' WHEN median_number_of_deployment_days_per_month >= 1 THEN median_number_of_deployment_days_per_month || ' deployment days per month(medium)' WHEN median_number_of_deployment_days_per_month < 1 AND NOT is_collected IS NULL THEN median_number_of_deployment_days_per_month || ' deployment days per month(low)' ELSE 'N/A. Please check if you have collected deployments.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN median_number_of_deployment_days_per_week || ' deployment days per week(elite)' WHEN median_number_of_deployment_days_per_month >= 1 THEN median_number_of_deployment_days_per_month || ' deployment days per month(high)' WHEN median_number_of_deployment_days_per_six_months >= 1 THEN median_number_of_deployment_days_per_six_months || ' deployment days per six months(medium)' WHEN median_number_of_deployment_days_per_six_months < 1 AND NOT is_collected IS NULL THEN median_number_of_deployment_days_per_six_months || ' deployment days per six months(low)' ELSE 'N/A. Please check if you have collected deployments.' END ELSE 'Invalid dora report' END AS \"Deployment Frequency\" FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Deployment Frequency", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^median_change_lead_time$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 2: median lead time for changes */ WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished in the selected period */ SELECT DISTINCT pr.id, ppm.pr_cycle_time FROM pull_requests AS pr JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _median_change_lead_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats), _median_change_lead_time AS (/* use median PR cycle time as the median change lead time */ SELECT MAX(pr_cycle_time) AS median_change_lead_time FROM _median_change_lead_time_ranks WHERE ranks <= 0.5) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_change_lead_time < 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_change_lead_time < 30 * 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_change_lead_time >= 30 * 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)' ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_change_lead_time < 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_change_lead_time < 180 * 24 * 60 THEN ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_change_lead_time >= 180 * 24 * 60 THEN (ROUND(CAST(CAST(median_change_lead_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)')::TEXT ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END ELSE 'Invalid dora report' END AS median_change_lead_time FROM _median_change_lead_time", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Median Lead Time for Changes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^change_failure_rate$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 4: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate AS (SELECT CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments), _is_collected_data AS (SELECT CASE WHEN COUNT(i.id) = 0 AND COUNT(cdc.id) = 0 THEN 'No All' WHEN COUNT(i.id) = 0 THEN 'No Incidents' WHEN COUNT(cdc.id) = 0 THEN 'No Deployments' END AS is_collected FROM (SELECT 1) AS dummy LEFT JOIN incidents AS i ON 1 = 1 LEFT JOIN cicd_deployment_commits AS cdc ON 1 = 1) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.05 THEN ROUND(change_failure_rate * 100, 1) || '%(elite)' WHEN change_failure_rate <= 0.10 THEN ROUND(change_failure_rate * 100, 1) || '%(high)' WHEN change_failure_rate <= 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(medium)' WHEN change_failure_rate > 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.15 THEN ROUND(change_failure_rate * 100, 1) || '%(elite)' WHEN change_failure_rate <= 0.20 THEN ROUND(change_failure_rate * 100, 1) || '%(high)' WHEN change_failure_rate <= 0.30 THEN ROUND(change_failure_rate * 100, 1) || '%(medium)' WHEN change_failure_rate > 0.30 THEN ROUND(change_failure_rate * 100, 1) || '%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END ELSE 'Invalid dora report' END AS change_failure_rate FROM _change_failure_rate, \"_is_collected_data\"", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Change Failure Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^median_time_in_hour$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5), _metric_recovery_time_2023_report AS (SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_recovery_time < 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_recovery_time < 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_recovery_time < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_recovery_time >= 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)' ELSE 'N/A. Please check if you have collected deployments or incidents.' END END AS median_recovery_time FROM _median_recovery_time), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND $__timeFilter(i.resolution_date)), _median_mttr_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _median_mttr AS (SELECT MAX(lead_time_minutes) AS median_time_to_resolve FROM _median_mttr_ranks WHERE ranks <= 0.5), _metric_mttr_2021_report AS (SELECT CASE WHEN ('$dora_report') = '2021' THEN CASE WHEN median_time_to_resolve < 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_time_to_resolve < 24 * 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_time_to_resolve < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_time_to_resolve >= 7 * 24 * 60 THEN (ROUND(CAST(CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)')::TEXT ELSE 'N/A. Please check if you have collected incidents.' END END AS median_time_to_resolve FROM _median_mttr) SELECT median_recovery_time AS median_time_in_hour FROM _metric_recovery_time_2023_report WHERE ('$dora_report') = '2023' UNION SELECT median_time_to_resolve AS median_time_to_resolve FROM _metric_mttr_2021_report WHERE ('$dora_report') = '2021'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "${title_value}", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 2, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 1: Number of deployments per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, COUNT(cicd_deployment_id) AS deployment_count FROM (SELECT cdc.cicd_deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS _production_deployments GROUP BY 1) SELECT cm.month, CASE WHEN d.deployment_count IS NULL THEN 0 ELSE d.deployment_count END AS \"Deployment Count\" FROM calendar_months AS cm LEFT JOIN _deployments AS d ON cm.month = d.month WHERE $__timeFilter(month_timestamp)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Monthly deployments", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 6, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 2: median change lead time per month */ WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT pr.id, TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YY/MM') AS month, ppm.pr_cycle_time FROM pull_requests AS pr JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _find_median_clt_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats), _clt AS (SELECT month, MAX(pr_cycle_time) AS median_change_lead_time FROM _find_median_clt_each_month_ranks WHERE ranks <= 0.5 GROUP BY month) SELECT cm.month, CASE WHEN _clt.median_change_lead_time IS NULL THEN 0 ELSE CAST(_clt.median_change_lead_time AS NUMERIC) / NULLIF(60, 0) END AS \"Median Change Lead Time In Hour\" FROM calendar_months AS cm LEFT JOIN _clt ON cm.month = _clt.month WHERE $__timeFilter(month_timestamp)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Median Lead Time for Changes", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "change_failure_rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 5, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* Metric 4: change failure rate per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate_for_each_month AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments GROUP BY 1) SELECT cm.month, cfr.change_failure_rate AS \"Change Failure Rate\" FROM calendar_months AS cm LEFT JOIN _change_failure_rate_for_each_month AS cfr ON cm.month = cfr.month WHERE $__timeFilter(month_timestamp)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Change Failure Rate", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 9, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN commits AS c ON cdc.commit_sha = c.sha JOIN user_accounts AS ua ON c.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY deployment_finished_month ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT deployment_finished_month, MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5 GROUP BY deployment_finished_month), _metric_recovery_time_2023_report AS (SELECT cm.month, CASE WHEN m.median_recovery_time IS NULL THEN 0 ELSE CAST(m.median_recovery_time AS NUMERIC) / NULLIF(60, 0) END AS median_recovery_time_in_hour FROM calendar_months AS cm LEFT JOIN _median_recovery_time AS m ON cm.month = m.deployment_finished_month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: median time to restore service - MTTR */ AS (/* get the number of incidents created each month */ SELECT DISTINCT i.id, TO_CHAR(CAST(i.resolution_date AS TIMESTAMP), 'YY/MM') AS month, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE ('${team:csv}' = '' OR t.name::text = ANY(ARRAY[${team:singlequote}]::text[])) AND NOT i.lead_time_minutes IS NULL), _find_median_mttr_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _mttr AS (SELECT month, MAX(lead_time_minutes) AS median_time_to_resolve FROM _find_median_mttr_each_month_ranks WHERE ranks <= 0.5 GROUP BY month), _metric_mttr_2021_report AS (SELECT cm.month, CASE WHEN m.median_time_to_resolve IS NULL THEN 0 ELSE CAST(m.median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) END AS median_time_to_resolve_in_hour FROM calendar_months AS cm LEFT JOIN _mttr AS m ON cm.month = m.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)) SELECT cm.month, CASE WHEN '${dora_report}' = '2023' THEN mrt.median_recovery_time_in_hour WHEN '${dora_report}' = '2021' THEN mm.median_time_to_resolve_in_hour END AS \"${title_value} In Hours\" FROM calendar_months AS cm LEFT JOIN _metric_recovery_time_2023_report AS mrt ON cm.month = mrt.month LEFT JOIN _metric_mttr_2021_report AS mm ON cm.month = mm.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "${title_value}", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM teams", + "hide": 0, + "includeAll": true, + "label": "Team", + "multi": false, + "name": "team", + "options": [], + "query": "SELECT DISTINCT name FROM teams", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "multi": false, + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "DORA (by Team)", + "uid": "66YkL8y4z-pg", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-debug.json b/grafana/dashboards/postgresql/dora-debug.json new file mode 100644 index 00000000000..8dba4808f9c --- /dev/null +++ b/grafana/dashboards/postgresql/dora-debug.json @@ -0,0 +1,3863 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 44, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard is designed to validate the [DORA dashboard](/grafana/d/qNo8_0M4z/dora?orgId=1).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 20, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Check \"Deployment Frequency\"", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 64, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See the definition and calculation logic of [Deployment Frequency](https://devlake.apache.org/docs/Metrics/DeploymentFrequency)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n- Transformation Required: Define `deployments` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 1 + }, + "PRODUCTION": { + "color": "green", + "index": 0 + }, + "SUCCESS": { + "color": "green", + "index": 2 + }, + "This project is selected": { + "color": "green", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 6 + }, + "id": 16, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, result, environment, COUNT(DISTINCT cdc.id) AS deployment_commit_count, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployment_count FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE $__timeFilter(cdc.finished_date) GROUP BY 1, 2, 3, 4, 5", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 1. All cicd_deployment_commits (the rows with 3 green columns will be used in the following steps)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Deployment Frequency$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN 'On-demand(elite)' WHEN median_number_of_deployment_days_per_week >= 1 THEN 'Between once per day and once per week(high)' WHEN median_number_of_deployment_days_per_month >= 1 THEN 'Between once per week and once per month(medium)' WHEN median_number_of_deployment_days_per_month < 1 AND NOT is_collected IS NULL THEN 'Fewer than once per month(low)' ELSE 'N/A. Please check if you have collected deployments.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_number_of_deployment_days_per_week >= 5 THEN 'On-demand(elite)' WHEN median_number_of_deployment_days_per_month >= 1 THEN 'Between once per day and once per month(high)' WHEN median_number_of_deployment_days_per_six_months >= 1 THEN 'Between once per month and once every 6 months(medium)' WHEN median_number_of_deployment_days_per_six_months < 1 AND NOT is_collected IS NULL THEN 'Fewer than once per six months(low)' ELSE 'N/A. Please check if you have collected deployments.' END ELSE 'Invalid dora report' END AS \"Deployment Frequency\" FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 1 - Deployment Frequency", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 1 + }, + "PRODUCTION": { + "color": "green", + "index": 0 + }, + "SUCCESS": { + "color": "green", + "index": 2 + }, + "This project is selected": { + "color": "green", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 11 + }, + "id": 29, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, result, environment, COUNT(DISTINCT cdc.id) AS deployment_commit_count, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployment_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY 1, 2, 3, 4, 5", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 2. Find the number of successful production deployments in this project (Last column)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 15 + }, + "id": 34, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Number of deployments per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, COUNT(cicd_deployment_id) AS deployment_count FROM (SELECT cdc.cicd_deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS _production_deployments GROUP BY 1) SELECT cm.month, CASE WHEN d.deployment_count IS NULL THEN 0 ELSE d.deployment_count END AS \"Deployment Count\" FROM calendar_months AS cm LEFT JOIN _deployments AS d ON cm.month = d.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 2 - Monthly deployments", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 1 + }, + "PRODUCTION": { + "color": "green", + "index": 0 + }, + "SUCCESS": { + "color": "green", + "index": 2 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "deployment_id" + }, + "properties": [ + { + "id": "custom.width" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 16 + }, + "id": 49, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployment_commit_rank AS (SELECT pm.project_name, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, cdc.id, cdc.cicd_deployment_id, cdc.cicd_scope_id, result, environment, finished_date, ROW_NUMBER() OVER (PARTITION BY cdc.cicd_deployment_id ORDER BY finished_date DESC NULLS LAST) AS _deployment_commit_rank FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION') SELECT project_name, cicd_deployment_id AS deployment_id, id AS deployment_commit_id /* a deployment may have multiple deployment_commits */, result, environment, finished_date FROM _deployment_commit_rank WHERE _deployment_commit_rank = 1 AND $__timeFilter(finished_date)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 3. Use the last finished_date of deployment commits as the finished date of deployments in this project", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "Between once per month and once every 6 months": { + "color": "yellow", + "index": 1 + }, + "Between once per week and once per month": { + "color": "green", + "index": 2 + }, + "Fewer than once per six months": { + "color": "red", + "index": 0 + }, + "On-demand": { + "color": "purple", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "day" + }, + "properties": [ + { + "id": "custom.width" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 21 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployment_commit_rank AS (SELECT pm.project_name, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, cdc.id, cdc.cicd_deployment_id, cdc.cicd_scope_id, result, environment, finished_date, ROW_NUMBER() OVER (PARTITION BY cdc.cicd_deployment_id ORDER BY finished_date DESC NULLS LAST) AS _deployment_commit_rank FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION'), _deployments AS (SELECT project_name, cicd_deployment_id AS deployment_id, id AS deployment_commit_id /* a deployment may have multiple deployment_commits */, result, environment, finished_date FROM _deployment_commit_rank WHERE _deployment_commit_rank = 1 AND $__timeFilter(finished_date)) SELECT DISTINCT CAST(finished_date AS DATE) AS day, COUNT(1) AS deployment_count FROM _deployments GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4. Daily deployments in this project [To check Figure 1]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "Between once per month and once every 6 months": { + "color": "yellow", + "index": 1 + }, + "Between once per week and once per month": { + "color": "green", + "index": 2 + }, + "Fewer than once per six months": { + "color": "red", + "index": 0 + }, + "On-demand": { + "color": "purple", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "day" + }, + "properties": [ + { + "id": "custom.width" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 21 + }, + "id": 50, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployment_commit_rank AS (SELECT pm.project_name, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, cdc.id, cdc.cicd_deployment_id, cdc.cicd_scope_id, result, environment, finished_date, ROW_NUMBER() OVER (PARTITION BY cdc.cicd_deployment_id ORDER BY finished_date DESC NULLS LAST) AS _deployment_commit_rank FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION'), _deployments AS (SELECT project_name, cicd_deployment_id AS deployment_id, id AS deployment_commit_id /* a deployment may have multiple deployment_commits */, result, environment, finished_date FROM _deployment_commit_rank WHERE _deployment_commit_rank = 1 AND $__timeFilter(finished_date)), _deployment_days AS (SELECT DISTINCT CAST(finished_date AS DATE) AS day, COUNT(1) AS deployment_count FROM _deployments GROUP BY 1) SELECT TO_CHAR(CAST(day AS TIMESTAMP), 'YY/MM') AS month, SUM(deployment_count) AS monthly_deployment_counts FROM _deployment_days GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 5. Monthly deployments in this project [To check Figure 2]", + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 28, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Check \"Median Lead Time for Changes\"", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 72, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See the definition and calculation logic of [Median Lead Time for Changes](https://devlake.apache.org/docs/Metrics/LeadTimeForChanges)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Pull Requests` from GitHub PRs, GitLab MRs, BitBucket PRs, Azure DevOps PRs, etc.\n- Transformation Required: Define `deployments` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.\n- Validatation Steps below:\n - Step 1 - check the data integrity of PRs in table `pull_requests`\n - Step 2 - check the data integrity of deployment_commit in table `cicd_deployment_commits`\n - Step 3 - check if a deployment_commit is associated with the correct PR in table `project_pr_metrics`\n - Step 4 - check if metrics like PR Coding/Pickup/Review/Deploy/cycle time are correct in table `project_pr_metrics`\n - Step 5 - check if the median lead time for changes in each month is correct", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 254, 254, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "gradient", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 34 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, pr._raw_data_table, COUNT(*) AS total_number_of_PRs FROM pull_requests AS pr /* \tjoin project_pr_metrics prm on prm.id = pr.id */ JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND /* \tand pr.merged_date is not null */ /* \tand prm.pr_cycle_time is not null */ $__timeFilter(pr.created_date) GROUP BY 1, 2", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 1-1. Check if the total PR number in the selected project(s) is correct", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 40, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^median_change_lead_time$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 2: median lead time for changes */ WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished in the selected period */ SELECT DISTINCT pr.id, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _median_change_lead_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats), _median_change_lead_time AS (/* use median PR cycle time as the median change lead time */ SELECT MAX(pr_cycle_time) AS median_change_lead_time FROM _median_change_lead_time_ranks WHERE ranks <= 0.5) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN median_change_lead_time < 24 * 60 THEN 'Less than one day(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN 'Between one day and one week(high)' WHEN median_change_lead_time < 30 * 24 * 60 THEN 'Between one week and one month(medium)' WHEN median_change_lead_time >= 30 * 24 * 60 THEN 'More than one month(low)' ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN median_change_lead_time < 60 THEN 'Less than one hour(elite)' WHEN median_change_lead_time < 7 * 24 * 60 THEN 'Less than one week(high)' WHEN median_change_lead_time < 180 * 24 * 60 THEN 'Between one week and six months(medium)' WHEN median_change_lead_time >= 180 * 24 * 60 THEN 'More than six months(low)' ELSE 'N/A. Please check if you have collected deployments/pull_requests.' END ELSE 'Invalid dora report' END AS median_change_lead_time FROM _median_change_lead_time", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 3 - Median Lead Time for Changes", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": false, + "inspect": false + }, + "mappings": [ + { + "options": { + "This Project": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "merged_date" + }, + "properties": [ + { + "id": "color" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "time" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 0 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "project_name" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 16, + "x": 0, + "y": 39 + }, + "id": 53, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, pr.title /* pr.status, */, pr.author_name, pr.url, pr.merged_date, pr.created_date FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' WHERE pr.id /* ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) */ = '$pr_id' AND $__timeFilter(pr.created_date)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 1-2. Check if the attributes of the selected pull request is correct (Use the Pull Request URL filter above)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 38, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* Metric 2: median change lead time per month */ WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT pr.id, TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YY/MM') AS month, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _find_median_clt_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats), _clt AS (SELECT month, MAX(pr_cycle_time) AS median_change_lead_time FROM _find_median_clt_each_month_ranks WHERE ranks <= 0.5 GROUP BY month) SELECT cm.month, CASE WHEN _clt.median_change_lead_time IS NULL THEN 0 ELSE CAST(_clt.median_change_lead_time AS NUMERIC) / NULLIF(60, 0) END AS \"Median Change Lead Time In Hour\" FROM calendar_months AS cm LEFT JOIN _clt ON cm.month = _clt.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 4 - Median Lead Time for Changes", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 254, 254, 1)", + "mode": "fixed" + }, + "mappings": [ + { + "options": { + "from": 1, + "result": { + "color": "green", + "index": 0 + }, + "to": 10000000 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 42 + }, + "id": 12, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS \"No. of merged PRs in table.pull_requests\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"No. of PRs in table.project_pr_metrics\" FROM project_pr_metrics WHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "B", + "select": [ + [ + { + "params": [ + "blueprint_id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprint_labels", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 1-3. Check if the No. of records in project_pr_metrics is the same as the No. of MERGED PRs in the selected project(s)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 1 + }, + "PRODUCTION": { + "color": "green", + "index": 0 + }, + "SUCCESS": { + "color": "green", + "index": 2 + }, + "This project is selected": { + "color": "green", + "index": 3 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 46 + }, + "id": 68, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, result, environment, COUNT(DISTINCT cdc.id) AS deployment_commit_count, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployment_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY 1, 2, 3, 4, 5", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 2. Check if the number of successful production deployments in the selected project(s) is correct", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "This Project": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "merged_date" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "color" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 0 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "select_status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "This project is selected": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 51, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, pr.title /* pr.status, */, pr.url /* \tpr.author_name, */, pr.merged_date, pr.created_date FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' WHERE $__timeFilter(pr.created_date) /* ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) */", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Appendix 1 - All PRs in this project (Only the rows with 2 green columns should appear in project_pr_metrics)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "left", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "NO": { + "color": "red", + "index": 1 + }, + "YES": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 50 + }, + "id": 69, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* This query can be used to test if the column \"deployment_commit_id\" is associated with the correct PR */ WITH pr_merge_commits AS (SELECT ppm.id AS pr_id, ppm.deployment_commit_id AS id_1, pr.merge_commit_sha, ppm.project_name FROM project_pr_metrics AS ppm LEFT JOIN pull_requests AS pr ON ppm.id = pr.id WHERE ('${project:csv}' = '' OR ppm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND ppm.id = '$pr_id'), _deployment_commits AS (SELECT DISTINCT cdc1.id AS id_2, cdc1.prev_success_deployment_commit_id, cdc1.commit_sha AS new_commit_sha, cdc2.commit_sha AS old_commit_sha, cdc1.finished_date, cd.commit_sha AS deployed_commits FROM cicd_deployment_commits AS cdc1 LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc1.prev_success_deployment_commit_id = cdc2.id JOIN commits_diffs AS cd ON cdc1.commit_sha = cd.new_commit_sha AND COALESCE(cdc2.commit_sha, '') = cd.old_commit_sha JOIN project_mapping AS pm ON cdc1.cicd_scope_id = pm.row_id WHERE cdc1.result = 'SUCCESS' AND cdc1.environment = 'PRODUCTION' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))), _find_deployment_commit_id_from_pr AS (SELECT pmc.pr_id, pmc.id_1, pmc.merge_commit_sha, pmc.project_name, dc.id_2, dc.prev_success_deployment_commit_id, dc.deployed_commits, RANK() OVER (PARTITION BY pr_id ORDER BY dc.finished_date NULLS FIRST) AS deployment_rank FROM pr_merge_commits AS pmc LEFT JOIN _deployment_commits AS dc ON pmc.merge_commit_sha = dc.deployed_commits) SELECT pr_id, id_1 AS deployment_commit_id /* If \"id_1\" equals \"id_2\", then pass */, CASE WHEN id_1 = COALESCE(id_2, '') THEN 'YES' ELSE 'NO' END AS if_the_mapping_logic_is_correct /* \tid_2, */ FROM _find_deployment_commit_id_from_pr WHERE deployment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 3. Check if a `pull request` is associated with the correct `depoloyment commit` in table project_pr_metrics", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 52, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _pr_commit_ranks AS (SELECT pr.id, pr.created_date AS pr_created_date, prc.commit_sha, prc.commit_authored_date, ROW_NUMBER() OVER (PARTITION BY pr.id ORDER BY prc.commit_authored_date ASC NULLS FIRST) AS commit_rank FROM pull_requests AS pr LEFT JOIN pull_request_commits AS prc ON pr.id = prc.pull_request_id WHERE pr.id = '$pr_id') SELECT id, CEIL(CAST(EXTRACT(EPOCH FROM (pr_created_date - commit_authored_date)) AS NUMERIC) / NULLIF(60, 0)) AS \"PR coding time from PRs and commits\" /* commit_sha as first_commit_sha, */ /* commit_authored_date as first_commit_authored_date, */ FROM _pr_commit_ranks WHERE commit_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT id, pr_coding_time AS \"PR coding time from project_pr_metrics\" /* first_commit_sha, */ FROM project_pr_metrics WHERE id = '$pr_id'", + "refId": "B", + "select": [ + [ + { + "params": [ + "blueprint_id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprint_labels", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4-1. Check if PR coding time in project_pr_metrics is correct (Adopt the Pull Request filter above)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 54, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _pr_comment_ranks AS (SELECT pr.id AS pr_id, pr.created_date AS pr_created_date, prc.id AS review_id, prc.created_date AS review_created_date, ROW_NUMBER() OVER (PARTITION BY pr.id ORDER BY prc.created_date ASC NULLS FIRST) AS comment_rank FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id WHERE pr.id = '$pr_id' AND prc.account_id <> pr.author_id) SELECT pr_id, review_id AS first_review_id, pr_created_date, review_created_date AS first_review_time, CEIL(CAST(EXTRACT(EPOCH FROM (review_created_date - pr_created_date)) AS NUMERIC) / NULLIF(60, 0)) AS \"PR pickup time from pr_comments\" FROM _pr_comment_ranks WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT id, pr_pickup_time AS \"PR pickup time from project_pr_metrics\" /* first_review_id, */ FROM project_pr_metrics WHERE id = '$pr_id' AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "B", + "select": [ + [ + { + "params": [ + "blueprint_id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprint_labels", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4-2. Check if PR pickup time in project_pr_metrics is correct (Adopt the Pull Request filter above)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 55, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _pr_comment_ranks AS (SELECT pr.id AS pr_id, pr.merged_date AS pr_merged_date, prc.id AS review_id, prc.created_date AS review_created_date, ROW_NUMBER() OVER (PARTITION BY pr.id ORDER BY prc.created_date ASC NULLS FIRST) AS comment_rank_asc FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id WHERE pr.id = '$pr_id' AND prc.account_id <> pr.author_id) SELECT pr_id, review_id AS first_review_id, review_created_date AS first_review_time, CEIL(CAST(EXTRACT(EPOCH FROM (pr_merged_date - review_created_date)) AS NUMERIC) / NULLIF(60, 0)) AS \"PR review time from pr_comments\" FROM _pr_comment_ranks WHERE comment_rank_asc = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT id, pr_review_time AS \"PR review time from project_pr_metrics\" /* first_review_id, */ FROM project_pr_metrics WHERE id = '$pr_id' AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "B", + "select": [ + [ + { + "params": [ + "blueprint_id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprint_labels", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4-3. Check if PR review time in project_pr_metrics is correct (Adopt the Pull Request filter above)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 56, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT ppm.id AS pr_id, ppm.deployment_commit_id, CEIL(CAST(EXTRACT(EPOCH FROM (cdc.finished_date - pr.merged_date)) AS NUMERIC) / NULLIF(60, 0)) AS \"PR deploy time from cicd_deployment_commits\" FROM project_pr_metrics AS ppm LEFT JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id LEFT JOIN pull_requests AS pr ON ppm.id = pr.id WHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.\"environment\" = 'PRODUCTION' AND ppm.id = '$pr_id'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT id, pr_deploy_time AS \"PR deploy time from project_pr_metrics\" FROM project_pr_metrics WHERE id = '$pr_id' AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "B", + "select": [ + [ + { + "params": [ + "blueprint_id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprint_labels", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4-4. Check if PR deploy time in project_pr_metrics is correct (Adopt the Pull Request filter above)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 57, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT ppm.id, (pr_coding_time + CEIL(CAST(EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date)) AS NUMERIC) / NULLIF(60, 0)) + pr_deploy_time) AS \"PR cycle time from lower-level metrics\", ppm.\"pr_cycle_time\" AS \"PR cycle time from project_pr_metrics\" FROM project_pr_metrics AS ppm LEFT JOIN pull_requests AS pr ON ppm.id = pr.id WHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND pr.id = '$pr_id'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4-5. Check if PR cycle time in project_pr_metrics is correct (Adopt the Pull Request filter above)", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "ranks" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "from": 0, + "result": { + "color": "green", + "index": 0 + }, + "to": 0.5 + }, + "type": "range" + }, + { + "options": { + "from": 0.5, + "result": { + "color": "orange", + "index": 1 + }, + "to": 1 + }, + "type": "range" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "month" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 70, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YY/MM') AS month, pr.id, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _find_median_clt_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats) SELECT month, id, pr_cycle_time AS change_lead_time_in_minutes, CAST(pr_cycle_time AS NUMERIC) / NULLIF(60, 0) AS change_lead_time_in_hours, ranks FROM _find_median_clt_each_month_ranks", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 5 - check the median change lead time for each month in Figure 4 (Compare the change_lead_time with the max ranks in GREEN before the first occurrence of ORANGE in each month)", + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 74 + }, + "id": 26, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Check \"Change Failure Rate\" & \"${title_value}\"", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 66, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See the definition and calculation logic of [${title_value}](https://devlake.apache.org/docs/Metrics/MTTR)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 67, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- See the definition and calculation logic of [Change Failure Rate](https://devlake.apache.org/docs/Metrics/CFR)\n- Data Sources Required: \n - `Deployments` from Jenkins, GitLab CI, GitHub Action, BitBucket Pipelines, or Webhook, etc. \n - `Incidents` from Jira issues, GitHub issues, TAPD issues, PagerDuty Incidents, etc. \n- Transformation Required: Define `deployments` and `incidents` in [data transformations](https://devlake.apache.org/docs/Configuration/Tutorial#step-3---add-transformations-optional) while configuring the blueprint of a project.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "gradient", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "INCIDENT": { + "color": "green", + "index": 0 + }, + "This project is selected": { + "color": "green", + "index": 1 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 0, + "y": 83 + }, + "id": 31, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* get the incident created within the selected time period in the top-right corner */ SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, i._raw_data_table, i.type, COUNT(1) AS issue_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id AND pm.\"table\" = 'boards' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND /* \tand i.type = 'INCIDENT' */ $__timeFilter(i.created_date) GROUP BY pm.project_name, \"select_status\", i._raw_data_table, i.type", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 1. All types of issues in table.issues (rows with 2 green columns will be used to construct project_incident_deployment_relationships)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 83 + }, + "id": 42, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5), _is_collected_data AS (SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM _deployments) AND NOT EXISTS(SELECT 1 FROM incidents) THEN 'No deployments and incidents' WHEN NOT EXISTS(SELECT 1 FROM _deployments) THEN 'No Deployments' WHEN NOT EXISTS(SELECT 1 FROM incidents) THEN 'No Incidents' ELSE 'No incidents are mapped to deployments' END AS is_collected FROM _deployments AS d, _incidents_for_deployments AS i), _metric_recovery_time_2023_report AS (SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No deployments and incidents' THEN 'N/A. Please check if you have collected deployments and incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN median_recovery_time < 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(elite)' WHEN median_recovery_time < 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(high)' WHEN median_recovery_time < 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(medium)' WHEN median_recovery_time >= 7 * 24 * 60 THEN ROUND(CAST(CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) || '(low)' ELSE 'No data' END END AS median_recovery_time FROM _median_recovery_time, _is_collected_data), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.created_date)), _median_mttr_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _median_mttr AS (SELECT MAX(lead_time_minutes) AS median_time_to_resolve FROM _median_mttr_ranks WHERE ranks <= 0.5), _metric_mttr_2021_report AS (SELECT CASE WHEN ('$dora_report') = '2021' THEN CASE WHEN median_time_to_resolve < 60 THEN 'Less than one hour(elite)' WHEN median_time_to_resolve < 24 * 60 THEN 'Less than one day(high)' WHEN median_time_to_resolve < 7 * 24 * 60 THEN 'Between one day and one week(medium)' WHEN median_time_to_resolve >= 7 * 24 * 60 THEN 'More than one week(low)' ELSE 'N/A. Please check if you have collected incidents.' END END AS median_time_to_resolve FROM _median_mttr) SELECT median_recovery_time AS median_time_in_hour FROM _metric_recovery_time_2023_report WHERE ('$dora_report') = '2023' UNION SELECT median_time_to_resolve AS median_time_to_resolve FROM _metric_mttr_2021_report WHERE ('$dora_report') = '2021'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 5 - ${title_value}", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "gradient", + "type": "color-background" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "INCIDENT": { + "color": "green", + "index": 0 + }, + "This project is selected": { + "color": "green", + "index": 1 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 0, + "y": 89 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* get the incident created within the selected time period in the top-right corner */ SELECT pm.project_name, CASE WHEN ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) THEN 'This project is selected' ELSE 'Not Selected' END AS select_status, i._raw_data_table, COUNT(1) AS issue_count FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.created_date) GROUP BY pm.project_name, \"select_status\", i._raw_data_table", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 2. Number of Incidents in the selected project(s)", + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 15, + "w": 16, + "x": 0, + "y": 93 + }, + "id": 61, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "\n\nIn this case:\n\n- Deployment-1 maps to Incident-1\n- Deployment-3 maps to Incident-2 and Incident-3\n- Deployment-2,4,5 doesn't map to any Incident", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Deployment - Incident Mapping and CFR calculation logic", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hours", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "median_time_to_resolve_in_hour" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 93 + }, + "id": 46, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY deployment_finished_month ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT deployment_finished_month, MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5 GROUP BY deployment_finished_month), _metric_recovery_time_2023_report AS (SELECT cm.month, CASE WHEN m.median_recovery_time IS NULL THEN 0 ELSE CAST(m.median_recovery_time AS NUMERIC) / NULLIF(60, 0) END AS median_recovery_time_in_hour FROM calendar_months AS cm LEFT JOIN _median_recovery_time AS m ON cm.month = m.deployment_finished_month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: median time to restore service - MTTR */ AS (/* get the number of incidents created each month */ SELECT DISTINCT i.id, TO_CHAR(CAST(i.created_date AS TIMESTAMP), 'YY/MM') AS month, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT i.lead_time_minutes IS NULL), _find_median_mttr_each_month_ranks AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY month ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _mttr AS (SELECT month, MAX(lead_time_minutes) AS median_time_to_resolve FROM _find_median_mttr_each_month_ranks WHERE ranks <= 0.5 GROUP BY month), _metric_mttr_2021_report AS (SELECT cm.month, CASE WHEN m.median_time_to_resolve IS NULL THEN 0 ELSE CAST(m.median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) END AS median_time_to_resolve_in_hour FROM calendar_months AS cm LEFT JOIN _mttr AS m ON cm.month = m.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)) SELECT cm.month, CASE WHEN '${dora_report}' = '2023' THEN mrt.median_recovery_time_in_hour WHEN '${dora_report}' = '2021' THEN mm.median_time_to_resolve_in_hour END AS \"${title_value} In Hours\" FROM calendar_months AS cm LEFT JOIN _metric_recovery_time_2023_report AS mrt ON cm.month = mrt.month LEFT JOIN _metric_mttr_2021_report AS mm ON cm.month = mm.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 6 - ${title_value}", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 103 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN i.id = NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate AS (SELECT CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments), _is_collected_data AS (SELECT CASE WHEN COUNT(i.id) = 0 AND COUNT(cdc.id) = 0 THEN 'No All' WHEN COUNT(i.id) = 0 THEN 'No Incidents' WHEN COUNT(cdc.id) = 0 THEN 'No Deployments' END AS is_collected FROM (SELECT 1) AS dummy LEFT JOIN incidents AS i ON 1 = 1 LEFT JOIN cicd_deployment_commits AS cdc ON 1 = 1) SELECT CASE WHEN ('$dora_report') = '2023' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.05 THEN '0-5%(elite)' WHEN change_failure_rate <= 0.10 THEN '5%-10%(high)' WHEN change_failure_rate <= 0.15 THEN '10%-15%(medium)' WHEN change_failure_rate > 0.15 THEN '> 15%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END WHEN ('$dora_report') = '2021' THEN CASE WHEN is_collected = 'No All' THEN 'N/A. Please check if you have collected deployments/incidents.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN change_failure_rate <= 0.15 THEN '0-15%(elite)' WHEN change_failure_rate <= 0.20 THEN '16%-20%(high)' WHEN change_failure_rate <= 0.30 THEN '21%-30%(medium)' WHEN change_failure_rate > 0.30 THEN '> 30%(low)' ELSE 'N/A. Please check if you have collected deployments/incidents.' END ELSE 'Invalid dora report' END AS change_failure_rate FROM _change_failure_rate, \"_is_collected_data\"", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 7 - Change Failure Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 0 + }, + "INCIDENT": { + "color": "red", + "index": 1 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 8, + "x": 0, + "y": 108 + }, + "id": 58, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployments AS (SELECT DISTINCT d.cicd_deployment_id AS deployment_id, d.result, d.environment, d.finished_date, d.cicd_scope_id, pm.project_name FROM cicd_deployment_commits AS d JOIN project_mapping AS pm ON d.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE d.result /* only result needs to specified, not environment */ = 'SUCCESS' /* choose your project_name */ AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))), _incidents AS (SELECT DISTINCT i.id AS issue_id, i.created_date, pm.project_name FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND i.\"table\" = pm.\"table\" WHERE pm.project_name /* choose your project_name */ IN ($project)) SELECT deployment_id AS id, 'DEPLOYMENT' AS type, finished_date AS time FROM _deployments UNION SELECT issue_id AS id, 'INCIDENT' AS type, created_date AS time FROM _incidents ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 3. The sequence of DEPLOYMENTS and INCIDENTS", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "FALSE": { + "color": "green", + "index": 1 + }, + "TRUE": { + "color": "red", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 8, + "x": 8, + "y": 108 + }, + "id": 59, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT cdc.cicd_deployment_id AS deployment_id /* in CFR we use deployment_commit_id as the deployment_id in a specific repo */, cdc.finished_date, pim.id AS incident_id, CASE WHEN NOT pim.id IS NULL THEN 'TRUE' ELSE 'FALSE' END AS has_failure FROM cicd_deployment_commits AS cdc LEFT JOIN project_incident_deployment_relationships AS pim ON cdc.cicd_deployment_id = pim.deployment_id LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(cdc.finished_date) ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Step 4. Check if the DEPLOYMENT and INCIDENT mapping results are consistent with them in step 3 and figure 7&8", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "change_failure_rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 111 + }, + "id": 48, + "options": { + "barRadius": 0, + "barWidth": 0.6, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate per month */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2), _change_failure_rate_for_each_month AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YY/MM') AS month, CASE WHEN COUNT(deployment_id) IS NULL THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS change_failure_rate FROM _failure_caused_by_deployments GROUP BY 1) SELECT cm.month, cfr.change_failure_rate AS \"Change Failure Rate\" FROM calendar_months AS cm LEFT JOIN _change_failure_rate_for_each_month AS cfr ON cm.month = cfr.month WHERE month_timestamp BETWEEN CAST(TO_CHAR(CAST($__timeFrom() AS TIMESTAMP), 'YYYY-MM-01') AS DATE) AND CAST(TO_CHAR(CAST($__timeTo() AS TIMESTAMP), 'YYYY-MM-01') AS DATE)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Figure 8 - Change Failure Rate", + "type": "barchart" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT Url || '--' || id FROM pull_requests", + "hide": 0, + "includeAll": false, + "label": "Pull Request Url", + "multi": false, + "name": "pr_id", + "options": [], + "query": "SELECT Url || '--' || id FROM pull_requests", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "multi": false, + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Validation", + "uid": "KGkUnV-Vz-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-details-change-failure-rate.json b/grafana/dashboards/postgresql/dora-details-change-failure-rate.json new file mode 100644 index 00000000000..8191eceec3c --- /dev/null +++ b/grafana/dashboards/postgresql/dora-details-change-failure-rate.json @@ -0,0 +1,758 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 15, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Go Back", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora?orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard shows the details about [Change Failure Rate](https://devlake.apache.org/docs/Metrics/CFR) in DORA.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 2 + }, + "id": 86, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT i.id) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) SELECT CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) AS \"change_failure_rate\" FROM _failure_caused_by_deployments", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Change Failure Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "gradient", + "type": "color-background" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "FALSE": { + "color": "red", + "index": 1 + }, + "TRUE": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "incident_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 134 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deployment_finished_date" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 271 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "created_date" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields[\"metric_hidden\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 17, + "w": 20, + "x": 4, + "y": 2 + }, + "id": 85, + "links": [ + { + "title": "docs", + "url": "https://devlake.apache.org/docs/Metrics/CFR/#how-is-it-calculated" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id AS \"deployment_id\", d.deployment_finished_date, pim.id AS incident_id, i.title, i.url, i.url AS \"metric_hidden\", i.created_date /* i.resolution_date, */ FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN issues AS i ON pim.id = i.id WHERE NOT pim.id IS NULL ORDER BY 2 NULLS FIRST) SELECT * FROM _failure_caused_by_deployments", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. Deployments that have caused incidents(failure)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 4, + "x": 0, + "y": 8 + }, + "id": 84, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 3: change failure rate */ WITH _deployments AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused_by_deployments AS (/* calculate the number of incidents caused by each deployment */ SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT i.id) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) SELECT SUM(has_incident) AS \"incident count\", COUNT(deployment_id) AS \"deployment count\" FROM _failure_caused_by_deployments", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 0 + }, + "INCIDENT": { + "color": "red", + "index": 1 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 58, + "links": [ + { + "targetBlank": true, + "title": "See detail logic", + "url": "https://devlake.apache.org/docs/Metrics/CFR#how-is-it-calculated" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployments AS (SELECT DISTINCT d.cicd_deployment_id AS deployment_id, d.result, d.environment, d.finished_date, d.cicd_scope_id, pm.project_name FROM cicd_deployment_commits AS d JOIN project_mapping AS pm ON d.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE d.result /* only result needs to specified, not environment */ = 'SUCCESS' /* choose your project_name */ AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(d.finished_date)), _incidents AS (SELECT DISTINCT i.id AS issue_id, i.created_date, pm.project_name FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND i.\"table\" = pm.\"table\" WHERE pm.project_name /* choose your project_name */ IN ($project) AND $__timeFilter(i.created_date)) SELECT finished_date AS \"Time (Ascending)\", deployment_id AS \"Entity ID\", 'DEPLOYMENT' AS \"Entity Type (Deployment/Incident)\" FROM _deployments UNION SELECT created_date AS \"Time (Ascending)\", issue_id AS \"Entity ID\", 'INCIDENT' AS \"Entity Type (Deployment/Incident)\" FROM _incidents ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Deployment and incidents timeline", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "multi": false, + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Details - Change Failure Rate", + "uid": "Change-failure-rate-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-details-deployment-frequency.json b/grafana/dashboards/postgresql/dora-details-deployment-frequency.json new file mode 100644 index 00000000000..581f85d1ed4 --- /dev/null +++ b/grafana/dashboards/postgresql/dora-details-deployment-frequency.json @@ -0,0 +1,1335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 42, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Go Back", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora?orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard shows the details about [deployment frequency](https://devlake.apache.org/docs/Metrics/DeploymentFrequency) in DORA.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "fixed" + }, + "custom": { + "align": "auto", + "cellOptions": { + "mode": "basic", + "type": "color-background" + }, + "filterable": true, + "inspect": true + }, + "links": [], + "mappings": [ + { + "options": { + "DEPLOYMENT": { + "color": "green", + "index": 1 + }, + "PRODUCTION": { + "color": "green", + "index": 0 + }, + "SUCCESS": { + "color": "green", + "index": 2 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "deployment_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 256 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "project_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 122 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "result" + }, + "properties": [ + { + "id": "custom.width", + "value": 116 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "environment" + }, + "properties": [ + { + "id": "custom.width", + "value": 118 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "deployment_commit_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 295 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "display_title" + }, + "properties": [ + { + "id": "custom.width", + "value": 530 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 498 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields[\"metric_hidden\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 18, + "x": 0, + "y": 2 + }, + "id": 79, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _deployment_commit_rank AS (SELECT pm.project_name, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, cdc.id, cdc.display_title, cdc.url, cdc.cicd_deployment_id, cdc.cicd_scope_id, result, environment, finished_date, ROW_NUMBER() OVER (PARTITION BY cdc.cicd_deployment_id ORDER BY finished_date DESC NULLS LAST) AS _deployment_commit_rank FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION') SELECT project_name, cicd_deployment_id AS deployment_id, CASE WHEN display_title = '' THEN 'N/A' ELSE display_title END AS display_title, url, url AS metric_hidden, result /* a deployment may have multiple deployment_commits */ /* id as deployment_commit_id, */, environment, finished_date FROM _deployment_commit_rank WHERE _deployment_commit_rank = 1 AND $__timeFilter(finished_date) ORDER BY finished_date DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Deployment list in the selected project(s)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "day_range" + }, + "properties": [ + { + "id": "custom.width", + "value": 210 + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 6, + "x": 18, + "y": 2 + }, + "id": 75, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_days AS (/* Construct the last few calendar days within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _deployment_commit_rank AS (SELECT pm.project_name, CASE WHEN cdc._raw_data_table <> '' THEN cdc._raw_data_table ELSE cdc.cicd_scope_id END AS _raw_data_table, cdc.id, cdc.cicd_deployment_id, cdc.cicd_scope_id, result, environment, finished_date, ROW_NUMBER() OVER (PARTITION BY cdc.cicd_deployment_id ORDER BY finished_date DESC NULLS LAST) AS _deployment_commit_rank FROM cicd_deployment_commits AS cdc LEFT JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND result = 'SUCCESS' AND environment = 'PRODUCTION'), _deployments AS (SELECT project_name, cicd_deployment_id AS deployment_id, id AS deployment_commit_id /* a deployment may have multiple deployment_commits */, result, environment, finished_date FROM _deployment_commit_rank WHERE _deployment_commit_rank = 1 AND $__timeFilter(finished_date)) SELECT TO_CHAR(CAST(last_few_calendar_days.day AS TIMESTAMP), 'YYYY-MM-DD') AS day, COALESCE(COUNT(d.finished_date), 0) AS successful_deployments_to_prod FROM last_few_calendar_days LEFT JOIN _deployments AS d ON last_few_calendar_days.day = CAST(d.finished_date AS DATE) GROUP BY 1 ORDER BY 1 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. Daily deployments in the selected project(s)", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 7 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 74, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), calendar_weeks AS (SELECT DISTINCT CAST(CAST(day AS DATE) + -(EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS DATE) AS start_of_week FROM last_few_calendar_months ORDER BY 1 ASC NULLS FIRST) SELECT TO_CHAR(CAST(cw.start_of_week AS TIMESTAMP), 'MM/DD') || ' - ' || TO_CHAR(CAST(cw.start_of_week + INTERVAL '6 DAY' AS TIMESTAMP), 'MM/DD') AS week, days_deployed FROM calendar_weeks AS cw LEFT JOIN _days_weekly_deploy AS b ON cw.start_of_week = b.week", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Weekly deployments in the selected project(s)", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 76, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xField": "month", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy) SELECT TO_CHAR(CAST(month AS TIMESTAMP), 'YY/MM') AS month, days_deployed FROM _days_monthly_deploy", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4. Per month deployments in the selected project(s)", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 180 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 77, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy) SELECT TO_CHAR(CAST(month - INTERVAL '5 MONTH' AS TIMESTAMP), 'YY/MM') || ' ~ ' || TO_CHAR(CAST(month AS TIMESTAMP), 'YY/MM') AS month_range, days_deployed_per_six_months AS days_deployed FROM _days_six_months_deploy ORDER BY month NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5. Per six months deployments in the selected project(s)", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 37 + }, + "id": 93, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# 2023 Benchmarks\n- Elite: >= 5 days per week\n- High: >= 1 day per week\n- Medium: >= 1 day per month\n- Low: < 1 day per month\n# 2021 Benchmarks\n- Elite: >= 5 days per week\n- High: >= 1 day per month\n- Medium: >= 1 day per six months\n- Low: < 1 day per six months", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "title": "DORA Report", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#a6a6a6", + "value": null + } + ] + }, + "unit": "days per week" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 37 + }, + "id": 90, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5), _tolal_production_deployment_days AS (SELECT COUNT(*) AS tpdd, COUNT(DISTINCT day) AS dtpdd FROM _production_deployment_days) SELECT median_number_of_deployment_days_per_week AS \"Median weekly deployment days\" /* tpdd as \"Production Deploy Counts\", */ /* dtpdd as \"Production Deployment Days\", */ /* median_number_of_deployment_days_per_month as \"Median monthly deployment days\", */ /* median_number_of_deployment_days_per_six_months as \"Median semi-annual deployment days\" */ FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months, _tolal_production_deployment_days", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "6. Median deployments days at the weekly level", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#a6a6a6", + "value": null + } + ] + }, + "unit": "days per month" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 37 + }, + "id": 91, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5), _tolal_production_deployment_days AS (SELECT COUNT(*) AS tpdd, COUNT(DISTINCT day) AS dtpdd FROM _production_deployment_days) SELECT median_number_of_deployment_days_per_month AS \"Median monthly deployment days\" /* tpdd as \"Production Deploy Counts\", */ /* dtpdd as \"Production Deployment Days\", */ /* median_number_of_deployment_days_per_week as \"Median weekly deployment days\" */ /* median_number_of_deployment_days_per_six_months as \"Median semi-annual deployment days\" */ FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months, _tolal_production_deployment_days", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "7. Median deployments days at monthly level", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "pattern": ".*elite.*", + "result": { + "color": "purple", + "index": 0 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*high.*", + "result": { + "color": "green", + "index": 1 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*medium.*", + "result": { + "color": "yellow", + "index": 2 + } + }, + "type": "regex" + }, + { + "options": { + "pattern": ".*low.*", + "result": { + "color": "red", + "index": 3 + } + }, + "type": "regex" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#a6a6a6", + "value": null + } + ] + }, + "unit": "days per six months" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 37 + }, + "id": 92, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Metric 1: Deployment Frequency */ WITH last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS day FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _production_deployment_days AS (/* When deploying multiple commits in one pipeline, GitLab and BitBucket may generate more than one deployment. However, DevLake consider these deployments as ONE production deployment and use the last one's finished_date as the finished date. */ SELECT cdc.cicd_deployment_id AS deployment_id, MAX(CAST(cdc.finished_date AS DATE)) AS day FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1), _days_weekly_deploy AS (/* calculate the number of deployment days every week */ SELECT CAST(last_few_calendar_months.day + -(EXTRACT(ISODOW FROM last_few_calendar_months.day) - 1) * INTERVAL '1 day' AS DATE) AS week, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS weeks_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY week), _days_monthly_deploy AS (/* calculate the number of deployment days every month */ SELECT CAST(last_few_calendar_months.day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(last_few_calendar_months.day AS DATE)) + 1) AS DATE) AS month, MAX(CASE WHEN NOT _production_deployment_days.day IS NULL THEN 1 ELSE NULL END) AS months_deployed, COUNT(DISTINCT _production_deployment_days.day) AS days_deployed FROM last_few_calendar_months LEFT JOIN _production_deployment_days ON _production_deployment_days.day = last_few_calendar_months.day GROUP BY month), _days_six_months_deploy AS (SELECT month, SUM(days_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS days_deployed_per_six_months, COUNT(months_deployed) OVER (ORDER BY month NULLS FIRST ROWS BETWEEN 5 PRECEDING AND CURRENT ROW) AS months_deployed_count, ROW_NUMBER() OVER (PARTITION BY ((EXTRACT(YEAR FROM CAST(month AS TIMESTAMP)) * 12 + EXTRACT(MONTH FROM CAST(month AS TIMESTAMP))) / 6) ORDER BY month DESC NULLS LAST) AS rn FROM _days_monthly_deploy), _median_number_of_deployment_days_per_week_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_weekly_deploy), _median_number_of_deployment_days_per_week AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_week FROM _median_number_of_deployment_days_per_week_ranks WHERE ranks <= 0.5), _median_number_of_deployment_days_per_month_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed NULLS FIRST) AS ranks FROM _days_monthly_deploy), _median_number_of_deployment_days_per_month AS (SELECT MAX(days_deployed) AS median_number_of_deployment_days_per_month FROM _median_number_of_deployment_days_per_month_ranks WHERE ranks <= 0.5), _days_per_six_months_deploy_by_filter AS (SELECT month, days_deployed_per_six_months, months_deployed_count FROM _days_six_months_deploy WHERE rn % 6 = 1), _median_number_of_deployment_days_per_six_months_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY days_deployed_per_six_months NULLS FIRST) AS ranks FROM _days_per_six_months_deploy_by_filter), _median_number_of_deployment_days_per_six_months AS (SELECT MIN(days_deployed_per_six_months) AS median_number_of_deployment_days_per_six_months, MIN(months_deployed_count) AS is_collected FROM _median_number_of_deployment_days_per_six_months_ranks WHERE ranks >= 0.5), _tolal_production_deployment_days AS (SELECT COUNT(*) AS tpdd, COUNT(DISTINCT day) AS dtpdd FROM _production_deployment_days) SELECT median_number_of_deployment_days_per_six_months AS \"Median semi-annual deployment days\" /* tpdd as \"Production Deploy Counts\", */ /* dtpdd as \"Production Deployment Days\", */ /* median_number_of_deployment_days_per_week as \"Median weekly deployment days\" */ /* median_number_of_deployment_days_per_month as \"Median monthly deployment days\" */ FROM _median_number_of_deployment_days_per_week, _median_number_of_deployment_days_per_month, _median_number_of_deployment_days_per_six_months, _tolal_production_deployment_days", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "8. Median deployments days at per six months level", + "type": "stat" + } + ], + "schemaVersion": 39, + "tags": [ + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "multi": false, + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Details - Deployment Frequency", + "uid": "Deployment-frequency-pg", + "version": 8, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-details-failed-deployment-recovery-time.json b/grafana/dashboards/postgresql/dora-details-failed-deployment-recovery-time.json new file mode 100644 index 00000000000..6962e3be8be --- /dev/null +++ b/grafana/dashboards/postgresql/dora-details-failed-deployment-recovery-time.json @@ -0,0 +1,478 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 45, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Go Back ", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora?orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard shows the details about [Failed Deployment Recovery Time\n](https://devlake.apache.org/docs/Metrics/MTTR) in DORA.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 0, + "y": 2 + }, + "id": 87, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _recovery_time_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY (EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60) NULLS FIRST) AS ranks FROM _incidents_for_deployments), _median_recovery_time AS (SELECT MAX((EXTRACT(EPOCH FROM (incident_resolution_date - deployment_finished_date))/60)) AS median_recovery_time FROM _recovery_time_ranks WHERE ranks <= 0.5), _is_collected_data AS (SELECT CASE WHEN NOT EXISTS(SELECT 1 FROM _deployments) AND NOT EXISTS(SELECT 1 FROM incidents) THEN 'No deployments and incidents' WHEN NOT EXISTS(SELECT 1 FROM _deployments) THEN 'No Deployments' WHEN NOT EXISTS(SELECT 1 FROM incidents) THEN 'No Incidents' ELSE 'No incidents are mapped to deployments' END AS is_collected FROM _deployments AS d, _incidents_for_deployments AS i) SELECT CASE WHEN is_collected = 'No deployments and incidents' THEN 'N/A. Please check if you have collected deployments and incidents.' WHEN is_collected = 'No Deployments' THEN 'N/A. Please check if you have collected deployments.' WHEN is_collected = 'No Incidents' THEN 'N/A. Please check if you have collected incidents.' WHEN NOT median_recovery_time IS NULL THEN (CAST(median_recovery_time AS NUMERIC) / NULLIF(60, 0))::TEXT ELSE 'No data' END AS median_recovery_time_in_hours FROM _median_recovery_time, _is_collected_data", + "refId": "D", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "1. Failed Deployment Recovery Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": true + }, + "mappings": [ + { + "options": { + "FALSE": { + "color": "red", + "index": 1 + }, + "TRUE": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "deployment_finished_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 174 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "incident_resolution_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 208 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_deployment_recovery_time" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "green", + "value": 24 + }, + { + "color": "#EAB839", + "value": 168 + }, + { + "color": "red", + "value": 720 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 19, + "x": 5, + "y": 2 + }, + "id": 80, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "failed_deployment_recovery_time" + ], + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "deployment_finished_date" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT fd.deployment_id AS \"deployment_id\", fd.deployment_finished_date, i.id AS incident_caused_by_deployment /* date_format(fd.deployment_finished_date, '%y/%m') as deployment_finished_month, */, i.title AS incident_title, i.url AS incident_url, i.url AS \"metric_hidden\", i.resolution_date AS incident_resolution_date /* i.created_date as incident_create_date, */, (EXTRACT(EPOCH FROM (i.resolution_date - fd.deployment_finished_date))/3600) AS \"failed_deployment_recovery_time\" FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)) SELECT * FROM _incidents_for_deployments WHERE NOT incident_resolution_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Deployments and Incidents Details", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 0, + "y": 12 + }, + "id": 88, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2023 report ***** -- */ /* Metric 4: Failed deployment recovery time */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents_for_deployments AS (SELECT i.id AS incident_id, i.created_date AS incident_create_date, i.resolution_date AS incident_resolution_date, fd.deployment_id AS caused_by_deployment, fd.deployment_finished_date, TO_CHAR(CAST(fd.deployment_finished_date AS TIMESTAMP), 'YY/MM') AS deployment_finished_month FROM incidents AS i LEFT JOIN project_incident_deployment_relationships AS pim ON i.id = pim.id JOIN _deployments AS fd ON pim.deployment_id = fd.deployment_id WHERE $__timeFilter(i.resolution_date)), _incidents /* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND i.\"table\" = pm.\"table\" AND pm.\"table\" = 'boards' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.created_date)) SELECT COUNT(incident_id) AS total_count FROM _incidents_for_deployments", + "refId": "D", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "2. The number of incidents caused by deployments", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Details - Failed Deployment Recovery Time", + "uid": "Failed-deployment-recovery-time-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json b/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json new file mode 100644 index 00000000000..6269f469d4e --- /dev/null +++ b/grafana/dashboards/postgresql/dora-details-lead-timefor-changes.json @@ -0,0 +1,967 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Go Back", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora?orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- This dashboard shows the details about [Lead Time for Changes](https://devlake.apache.org/docs/Metrics/LeadTimeForChanges) in DORA.\n- It displays the statistics for pull requests **deployed** within the selected time range.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 75, + "links": [ + { + "targetBlank": true, + "title": "PR Cycle Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCycleTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.created_date AS pr_issued_date, COALESCE(CAST(prm.pr_cycle_time AS NUMERIC) / NULLIF(60, 0), 0) AS cycle_time /* convert null to 0 if a PR has no cycle_time to make sure cycle_time equals the sum of the four metrics below */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(prm.pr_deployed_date) AND /* and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT AVG(cycle_time) AS \"PR Cycle Time(h)\" FROM _prs", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1. Average PR Cycle Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 2 + }, + "id": 71, + "links": [ + { + "targetBlank": true, + "title": "PR Coding Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCodingTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.created_date AS pr_issued_date, COALESCE(CAST(prm.pr_coding_time AS NUMERIC) / NULLIF(60, 0), 0) AS coding_time /* convert null to 0 if a PR has no coding_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(prm.pr_deployed_date) AND /* and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT AVG(coding_time) AS \"Coding Time(h)\" FROM _prs", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Average PR Coding Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 2 + }, + "id": 72, + "links": [ + { + "targetBlank": true, + "title": "PR Pickup Time", + "url": "https://devlake.apache.org/docs/Metrics/PRPickupTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.created_date AS pr_issued_date, COALESCE(CAST(prm.pr_pickup_time AS NUMERIC) / NULLIF(60, 0), 0) AS pickup_time /* convert null to 0 if a PR has no pickup_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(prm.pr_deployed_date) AND /* and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT AVG(pickup_time) AS \"Pickup Time(h)\" FROM _prs", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.2 Average PR Pickup Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 2 + }, + "id": 73, + "links": [ + { + "targetBlank": true, + "title": "PR Review Time", + "url": "https://devlake.apache.org/docs/Metrics/PRReviewTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.created_date AS pr_issued_date, COALESCE(CAST(prm.pr_review_time AS NUMERIC) / NULLIF(60, 0), 0) AS review_time /* convert null to 0 if a PR has no review_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(prm.pr_deployed_date) AND /* and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT AVG(review_time) AS \"Review Time(h)\" FROM _prs", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.3 Average PR Review Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 2 + }, + "id": 74, + "links": [ + { + "targetBlank": true, + "title": "PR Deploy Time", + "url": "https://devlake.apache.org/docs/Metrics/PRDeployTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.created_date AS pr_issued_date, COALESCE(CAST(prm.pr_deploy_time AS NUMERIC) / NULLIF(60, 0), 0) AS deploy_time /* convert null to 0 if a PR has no deploy_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(prm.pr_deployed_date) AND /* and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT AVG(deploy_time) AS \"PR Deploy Time(h)\" FROM _prs", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.4 Average PR Deploy Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Lead time for changes = median(PR deployed date - PR's first commit's authored date)", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "green", + "value": 24 + }, + { + "color": "orange", + "value": 168 + }, + { + "color": "red", + "value": 720 + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "change_lead_time" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR url" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields[\"metric_hidden\"]}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR title" + }, + "properties": [ + { + "id": "custom.width", + "value": 674 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 70, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": true, + "enablePagination": false, + "fields": "", + "reducer": [ + "count" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _pr_stats AS (/* get the cycle time of PRs deployed by the deployments finished each month */ SELECT DISTINCT pr.id, pr.title, pr.url, pr.created_date, ppm.pr_coding_time, ppm.pr_pickup_time, ppm.pr_review_time, ppm.pr_deploy_time, ppm.first_commit_sha, prc.commit_authored_date, cdc.cicd_deployment_id, cdc.name, cdc.finished_date, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id JOIN pull_request_commits AS prc ON prc.commit_sha = ppm.first_commit_sha WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)) SELECT title AS \"PR title\" /* id as \"PR id\", */, url AS \"PR url\", url AS metric_hidden, first_commit_sha AS \"First commit sha\" /* created_date as \"PR created_date\", */, commit_authored_date AS \"First commit authored date\", cicd_deployment_id AS \"Deployment id\", finished_date AS \"Deployment finished_date\" /* name as \"Deployment name\", */, CAST(pr_coding_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_coding_time\", CAST(pr_pickup_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_pickup_time\", CAST(pr_review_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_review_time\", CAST(pr_deploy_time AS NUMERIC) / NULLIF(60, 0) AS \"pr_deploy_time\", CAST(pr_cycle_time AS NUMERIC) / NULLIF(60, 0) AS change_lead_time FROM _pr_stats", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2. PR Details", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT dora_report FROM dora_benchmarks", + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [], + "query": "SELECT dora_report FROM dora_benchmarks", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Failed Deployment Recovery Time", + "value": "Failed Deployment Recovery Time" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "hide": 2, + "includeAll": false, + "label": "TitleValue", + "multi": false, + "name": "title_value", + "options": [], + "query": "SELECT CASE WHEN dora_report = '2023' THEN 'Failed Deployment Recovery Time' WHEN dora_report = '2021' THEN 'Median Time to Restore Service' ELSE NULL END AS title_value FROM dora_benchmarks WHERE dora_report = '${dora_report:raw}'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Details - Lead Time for Changes", + "uid": "Lead-time-for-changes-pg", + "version": 7, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/dora-details-timeto-restore-service.json b/grafana/dashboards/postgresql/dora-details-timeto-restore-service.json new file mode 100644 index 00000000000..be9063c40db --- /dev/null +++ b/grafana/dashboards/postgresql/dora-details-timeto-restore-service.json @@ -0,0 +1,465 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 10, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Go Back", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora?orgId=1" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 63, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "This dashboard shows the details about [Time to restore service\n](https://devlake.apache.org/docs/Metrics/MTTR) in DORA.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 5, + "x": 0, + "y": 2 + }, + "id": 87, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.resolution_date)), _median_mttr_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS ranks FROM _incidents), _median_mttr AS (SELECT MAX(lead_time_minutes) AS median_time_to_resolve FROM _median_mttr_ranks WHERE ranks <= 0.5) SELECT CAST(median_time_to_resolve AS NUMERIC) / NULLIF(60, 0) AS median_time_to_resolve_in_hours FROM _median_mttr", + "refId": "D", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "1. Median Time to Restore Service", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "rgba(255, 255, 255, 1)", + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [ + { + "options": { + "FALSE": { + "color": "red", + "index": 1 + }, + "TRUE": { + "color": "green", + "index": 0 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "green", + "value": 24 + }, + { + "color": "yellow", + "value": 168 + }, + { + "color": "red", + "value": 720 + } + ] + }, + "unit": "h" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields[\"metric_hidden\"]}" + } + ] + }, + { + "id": "custom.width", + "value": 432 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "metric_hidden" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 819 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "time_to_restore_service" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 19, + "x": 5, + "y": 2 + }, + "id": 86, + "links": [], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": [ + "lead_time_minutes" + ], + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "resolution_date" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ WITH _incidents AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id AS \"incident_id\", i.title, i.url, i.url AS \"metric_hidden\", i.resolution_date /* i.created_date, */, CAST(CAST(lead_time_minutes AS NUMERIC) / NULLIF(60, 0) AS BIGINT) AS time_to_restore_service FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.resolution_date)) SELECT * FROM _incidents ORDER BY resolution_date DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_tasks", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3. Time to restore service details", + "transparent": true, + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 0, + "y": 11 + }, + "id": 88, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "/* ***** 2021 report ***** -- */ /* Metric 4: Median time to restore service */ WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _incidents AS (/* get the incidents created within the selected time period in the top-right corner */ SELECT DISTINCT i.id, CAST(lead_time_minutes AS BIGINT) AS lead_time_minutes FROM incidents AS i JOIN project_mapping AS pm ON i.scope_id = pm.row_id AND pm.\"table\" = i.\"table\" WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.created_date)) SELECT COUNT(id) AS \"incident count\" FROM _incidents", + "refId": "D", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "2. Incident Count", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "DORA" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "DORA Details - Time to Restore Service", + "uid": "Time-to-restore-service-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/engineering-overview.json b/grafana/dashboards/postgresql/engineering-overview.json new file mode 100644 index 00000000000..6d22a879f9c --- /dev/null +++ b/grafana/dashboards/postgresql/engineering-overview.json @@ -0,0 +1,2386 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 40, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 32, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard is to overview the Git and project management metrics.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira)", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 10 + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND ('${priority:csv}' = '' OR i.priority::text = ANY(ARRAY[${priority:singlequote}]::text[])) AND i.type = 'BUG' AND CAST(i.created_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Critical Defects Identified [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 3 + }, + "id": 22, + "options": { + "barRadius": 0, + "barWidth": 0.46, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, COUNT(DISTINCT i.id) AS defect_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND ('${priority:csv}' = '' OR i.priority::text = ANY(ARRAY[${priority:singlequote}]::text[])) AND i.type = 'BUG' AND $__timeFilter(i.created_date) AND i.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY time) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, defect_count FROM _issues ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Critical Defects Identified over Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 11 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND ('${type:csv}' = '' OR i.type::text = ANY(ARRAY[${type:singlequote}]::text[])) AND i.status = 'DONE' AND i.resolution_date BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Issue Lead Time in Days [Issues Resolved in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lead Time(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 11 + }, + "id": 23, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS issue_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND i.resolution_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, issue_lead_time AS \"Mean Requirement Lead Time in Days\" FROM _issues ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Requirement Lead Time in Days [Of Issues Resolved per Month]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "green", + "value": 15 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT author_name) FROM commits AS c JOIN repo_commits AS rc ON c.sha = rc.commit_sha JOIN project_mapping AS pm ON rc.repo_id = pm.row_id AND pm.table = 'repos' WHERE CAST(authored_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Developers [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 19 + }, + "id": 24, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _developers AS (SELECT CAST(c.authored_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(c.authored_date AS DATE)) + 1) AS time, COUNT(DISTINCT author_name) AS developer_count FROM commits AS c JOIN repo_commits AS rc ON c.sha = rc.commit_sha JOIN project_mapping AS pm ON rc.repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(c.authored_date) AND c.authored_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY time) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, developer_count FROM _developers ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Developers", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 60 + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _num_issues_with_sprint_updated AS (SELECT NULLIF(COUNT(DISTINCT i.id), 0) AS num_issues_with_sprint_updated FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN issue_changelogs AS c ON i.id = c.issue_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND c.field_name = 'Sprint' AND c.original_from_value <> '' AND c.original_to_value <> '' AND CAST(i.created_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'), _total_num_issues AS (SELECT NULLIF(COUNT(DISTINCT i.id), 0) AS total_num_issues FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH') SELECT NOW() AS time, 100 - CAST(100 * (SELECT 1.0 * num_issues_with_sprint_updated FROM _num_issues_with_sprint_updated) AS NUMERIC) / NULLIF((SELECT total_num_issues FROM _total_num_issues), 0) AS ratio", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "On-time Delivery [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 27 + }, + "id": 25, + "links": [ + { + "targetBlank": true, + "title": "Requirement Delivery Rate", + "url": "https://devlake.apache.org/docs/Metrics/RequirementDeliveryRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _num_issues_with_sprint_updated AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, NULLIF(COUNT(DISTINCT i.id), 0) AS num_issues_with_sprint_updated FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN issue_changelogs AS c ON i.id = c.issue_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND c.field_name = 'Sprint' AND c.original_from_value <> '' AND c.original_to_value <> '' AND $__timeFilter(i.created_date) AND i.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY time), _total_num_issues AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, NULLIF(COUNT(DISTINCT i.id), 0) AS total_num_issues FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND i.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY time) SELECT x.time, 100 - 100 * (CAST(1.0 * x.num_issues_with_sprint_updated AS NUMERIC) / NULLIF(y.total_num_issues, 0)) AS delivery_rate FROM _num_issues_with_sprint_updated AS x JOIN _total_num_issues AS y ON x.time = y.time", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "On-time Delivery over Month", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 35 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE NOT pr.merged_date IS NULL AND CAST(pr.merged_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs merged [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 35 + }, + "id": 26, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _merged_prs AS (SELECT CAST(pr.merged_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.merged_date AS DATE)) + 1) AS time, COUNT(DISTINCT pr.id) AS pr_merged_count FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.merged_date) AND pr.merged_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY time) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, pr_merged_count FROM _merged_prs ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Merged over Month", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "orange", + "value": 20 + }, + { + "color": "red", + "value": 40 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 43 + }, + "id": 16, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND CAST(pr.created_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Unlinked PRs % [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 43 + }, + "id": 28, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\"", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Unlinked PRs Ratio over Month", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _commits_groupby_name_and_date AS (SELECT author_name, CAST(authored_date AS DATE) AS _day, COUNT(DISTINCT c.sha) FROM commits AS c JOIN repo_commits AS rc ON c.sha = rc.commit_sha JOIN project_mapping AS pm ON rc.repo_id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND (EXTRACT(ISODOW FROM authored_date) - 1) BETWEEN 0 AND 4 AND CAST(authored_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH' GROUP BY \"author_name\", CAST(authored_date AS DATE)) SELECT CAST(100 * COUNT(*) AS NUMERIC) / NULLIF((COUNT(DISTINCT author_name) * COUNT(DISTINCT _day)), 0) FROM _commits_groupby_name_and_date", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Coding Days % [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 51 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _commits_groupby_name_and_date AS (SELECT author_name, CAST(authored_date AS DATE) AS _day, COUNT(DISTINCT c.sha) FROM commits AS c JOIN repo_commits AS rc ON c.sha = rc.commit_sha JOIN project_mapping AS pm ON rc.repo_id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND ((EXTRACT(ISODOW FROM authored_date) - 1) BETWEEN 0 AND 4) AND $__timeFilter(authored_date) AND authored_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1, 2) SELECT _day + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(_day AS DATE)) + 1) AS time, CAST(100 * COUNT(*) AS NUMERIC) / NULLIF((COUNT(DISTINCT author_name) * COUNT(DISTINCT _day)), 0) AS working_days_percentatages_per_month FROM _commits_groupby_name_and_date GROUP BY time", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Percentage of Coding Days Each Month", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 59 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND CAST(pr.created_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_raw_github_api_repositories", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Time to Merge [in $month]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 10, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 59 + }, + "id": 30, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS pr_time_to_merge_in_days FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.created_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\" ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_raw_github_api_repositories", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean Time to Merge over Month", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 21, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT CASE WHEN i.type = 'BUG' THEN i.id ELSE NULL END) AS \"Bug\", COUNT(DISTINCT CASE WHEN i.type <> 'BUG' AND epic_key <> '' THEN i.id ELSE NULL END) AS \"Strategic\", COUNT(DISTINCT CASE WHEN i.type <> 'BUG' AND epic_key = '' THEN i.id ELSE NULL END) AS \"Non-Strategic\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT i.resolution_date IS NULL AND CAST(resolution_date AS DATE) BETWEEN TO_DATE('$month', 'YYYY-MM-DD') AND TO_DATE('$month', 'YYYY-MM-DD') + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Work Done % [in $month]", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 68, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 20, + "options": { + "barRadius": 0, + "barWidth": 0.27, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.priority AS \"Priority\", AVG(CAST((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS \"Average Age\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND i.status = 'TODO' AND i.type = 'BUG' AND ('${priority:csv}' = '' OR i.priority::text = ANY(ARRAY[${priority:singlequote}]::text[])) GROUP BY i.priority", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "_devlake_blueprints", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Age of Critical Outstanding Defects by Priority", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 75 + }, + "id": 34, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard", + "Highlights" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT (TO_CHAR(CAST(CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS TIMESTAMP), 'YYYY-MM') || ':' || TO_CHAR(CAST(CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS TIMESTAMP), 'YYYY-MM-DD')) AS month FROM issues AS i ORDER BY month DESC NULLS LAST", + "hide": 0, + "includeAll": false, + "label": "Selected Month", + "multi": false, + "name": "month", + "options": [], + "query": "SELECT DISTINCT (TO_CHAR(CAST(month_timestamp AS TIMESTAMP), 'YYYY-MM') || ':' || TO_CHAR(CAST(month_timestamp AS TIMESTAMP), 'YYYY-MM-DD')) AS month FROM calendar_months WHERE month_timestamp <= CURRENT_DATE ORDER BY month DESC NULLS LAST LIMIT 12", + "refresh": 1, + "regex": "/^(?.*):(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT priority FROM issues", + "description": "Customize what prioriti(es) are considered \"critical\"", + "hide": 0, + "includeAll": true, + "label": "Prioriti(es) of Critical Issues", + "multi": true, + "name": "priority", + "options": [], + "query": "SELECT DISTINCT priority FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "Engineering Overview", + "uid": "ZF6abXX7z-pg", + "version": 15, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time-team-view.json b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time-team-view.json new file mode 100644 index 00000000000..2e45236b144 --- /dev/null +++ b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time-team-view.json @@ -0,0 +1,3419 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 142, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira) ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is)\n- You also need to do [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 89, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Total and Avg PRs Opened/Merged", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Total PR Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Total PR Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 111, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team1: Total PR Opened\", COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team2: Total PR Opened\" FROM _prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total PRs Opened", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: PR Opened per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: PR Opened per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 110, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: PR Opened per Member\", CAST(COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: PR Opened per Member\", CAST(COUNT(DISTINCT id) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: PR Opened per Member\" FROM _prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Opened per Member", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Total PR Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Total PR Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) AND NOT merged_date IS NULL THEN id WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) AND merged_date IS NULL THEN NULL END) AS \"Team1: Total PR Merged\", COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) AND NOT merged_date IS NULL THEN id WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) AND merged_date IS NULL THEN NULL END) AS \"Team2: Total PR Merged\" FROM _prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total PRs Merged", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: PR Merged per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: PR Merged per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 112, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) AND NOT merged_date IS NULL THEN id WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) AND merged_date IS NULL THEN NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: PR Merged per Member\", CAST(COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) AND NOT merged_date IS NULL THEN id WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) AND merged_date IS NULL THEN NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: PR Merged per Member\", CAST(COUNT(DISTINCT CASE WHEN NOT merged_date IS NULL THEN id END) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: PR Merged per Member\" FROM _prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Merged per Member", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 114, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Issue Completed", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Issues Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Issues Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 74, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status = 'DONE' AND ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team1: Issues Completed\" /* count(i.id) AS \"Issues Opened\", */, COUNT(DISTINCT CASE WHEN status = 'DONE' AND ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team2: Issues Completed\" FROM _issues GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Issue Completed", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Issues Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Issues Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 115, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT DISTINCT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'DONE' AND ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: Issues Completed per Member\", CAST(COUNT(DISTINCT CASE WHEN status = 'DONE' AND ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: Issues Completed per Member\", CAST(COUNT(DISTINCT id) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: Issues Completed per Member\" FROM _issues GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Completed per Member", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 126, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "3. Story Points Completed", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Story Points Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Story Points Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT DISTINCT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, SUM(CASE WHEN status = 'DONE' AND ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN story_point ELSE 0 END) AS \"Team1: Story Points Completed\" /* count(i.id) AS \"Issues Opened\", */, SUM(CASE WHEN status = 'DONE' AND ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN story_point ELSE 0 END) AS \"Team2: Story Points Completed\" FROM _issues GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Story Points Completed", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Story Points Completed per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Story Points Completed per Member" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT DISTINCT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(SUM(CASE WHEN status = 'DONE' AND ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN story_point ELSE 0 END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: Story Points Completed per Member\", CAST(SUM(CASE WHEN status = 'DONE' AND ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN story_point ELSE 0 END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: Story Points Completed per Member\", CAST(COUNT(DISTINCT id) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: Story Points Completed per Member\" FROM _issues GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Story Points Completed per Member", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 124, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "4. PR Review Depth and PR Merged Size", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: PR Review Depth" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: PR Review Depth" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 122, + "links": [ + { + "targetBlank": true, + "title": "PR Review Depth", + "url": "https://devlake.apache.org/docs/Metrics/PRReviewDepth" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _merged_prs AS (SELECT DISTINCT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, prc.id AS comment_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN comment_id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: PR Review Depth\", CAST(COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN comment_id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: PR Review Depth\", CAST(COUNT(DISTINCT comment_id) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: PR Review Depth\" FROM _merged_prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": " Average PR Review Depth per Team", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: PR Size" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: PR Size" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 127, + "links": [ + { + "targetBlank": true, + "title": "PR Size", + "url": "https://devlake.apache.org/docs/Metrics/PRSize" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT DISTINCT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, pr.merge_commit_sha, c.additions + c.deletions AS loc, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN commits AS c ON pr.merge_commit_sha = c.sha JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND pr.status = 'MERGED' ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(SUM(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN loc ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: PR Size\", CAST(SUM(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN loc ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: PR Size\", CAST(SUM(loc) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: PR Merged Size\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": " Average PR Merged Size by Team", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 119, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "5. P0/P1 Bugs", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bug Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: P0/P1 Bugs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: P0/P1 Bugs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 100, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _bugs AS (SELECT DISTINCT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND i.type = 'BUG' ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team1: P0/P1 Bugs\", COUNT(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS \"Team2: P0/P1 Bugs\" FROM _bugs WHERE priority /* please choose the priorities in the filter above */ IN (${priority}) GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total P0/P1 Bugs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bug Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: P0/P1 Bugs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: P0/P1 Bugs" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _bugs AS (SELECT DISTINCT i.id, i.url, i.created_date, i.status, i.assignee_id, i.story_point, i.priority, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id JOIN user_accounts AS ua ON i.assignee_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND i.type = 'BUG' ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[]))), 0) AS \"Team1: P0/P1 Bugs per Member\", CAST(COUNT(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN id ELSE NULL END) AS NUMERIC) / NULLIF((SELECT COUNT(DISTINCT user_id) FROM team_users WHERE ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[]))), 0) AS \"Team2: P0/P1 Bugs per Member\", CAST(COUNT(DISTINCT id) AS NUMERIC) / NULLIF((SELECT COUNT(*) FROM users), 0) AS \"Org: P0/P1 Bugs per Member\" FROM _bugs WHERE priority /* please choose the priorities in the filter above */ IN (${priority}) GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "P0/P1 Bugs per Member", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 103, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "6. Cycle Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Avg Cycle Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Avg Cycle Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 136, + "links": [ + { + "targetBlank": true, + "title": "PR Cycle Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCycleTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_cycle_time AS NUMERIC) / NULLIF(60, 0), 0) AS cycle_time /* convert null to 0 if a PR has no cycle_time to make sure cycle_time equals the sum of the four metrics below */, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id LEFT JOIN user_accounts AS ua ON pr.author_id = ua.account_id LEFT JOIN users AS u ON ua.user_id = u.id LEFT JOIN team_users AS tu ON u.id = tu.user_id LEFT JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.merged_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN cycle_time END) AS \"Team1: Avg Cycle Time(h)\", AVG(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN cycle_time END) AS \"Team2: Avg Cycle Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Cycle Time(h)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Avg Coding Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Avg Coding Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 58 + }, + "id": 140, + "links": [ + { + "targetBlank": true, + "title": "PR Coding Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCodingTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_coding_time AS NUMERIC) / NULLIF(60, 0), 0) AS coding_time /* convert null to 0 if a PR has no coding_time to make sure cycle_time equals the sum of the four sub-metrics */, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id LEFT JOIN user_accounts AS ua ON pr.author_id = ua.account_id LEFT JOIN users AS u ON ua.user_id = u.id LEFT JOIN team_users AS tu ON u.id = tu.user_id LEFT JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.merged_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN coding_time END) AS \"Team1: Avg Coding Time(h)\", AVG(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN coding_time END) AS \"Team2: Avg Coding Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Coding Time(h)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Avg Pickup Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Avg Pickup Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 58 + }, + "id": 134, + "links": [ + { + "targetBlank": true, + "title": "PR Pickup Time", + "url": "https://devlake.apache.org/docs/Metrics/PRPickupTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_pickup_time AS NUMERIC) / NULLIF(60, 0), 0) AS pickup_time /* convert null to 0 if a PR has no pickup_time to make sure cycle_time equals the sum of the four sub-metrics */, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id LEFT JOIN user_accounts AS ua ON pr.author_id = ua.account_id LEFT JOIN users AS u ON ua.user_id = u.id LEFT JOIN team_users AS tu ON u.id = tu.user_id LEFT JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.merged_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN pickup_time END) AS \"Team1: Avg Pickup Time(h)\", AVG(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN pickup_time END) AS \"Team2: Avg Pickup Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Pickup Time(h)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Avg Review Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Avg Review Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 58 + }, + "id": 135, + "links": [ + { + "targetBlank": true, + "title": "PR Review Time", + "url": "https://devlake.apache.org/docs/Metrics/PRReviewTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_review_time AS NUMERIC) / NULLIF(60, 0), 0) AS review_time /* convert null to 0 if a PR has no review_time to make sure cycle_time equals the sum of the four sub-metrics */, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id LEFT JOIN user_accounts AS ua ON pr.author_id = ua.account_id LEFT JOIN users AS u ON ua.user_id = u.id LEFT JOIN team_users AS tu ON u.id = tu.user_id LEFT JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.merged_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN review_time END) AS \"Team1: Avg Review Time(h)\", AVG(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN review_time END) AS \"Team2: Avg Review Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Review Time(h)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: Avg Review Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: Avg Review Time(h)" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 58 + }, + "id": 145, + "links": [ + { + "targetBlank": true, + "title": "PR Deploy Time", + "url": "https://devlake.apache.org/docs/Metrics/PRDeployTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_deploy_time AS NUMERIC) / NULLIF(60, 0), 0) AS deploy_time /* convert null to 0 if a PR has no deploy_time to make sure cycle_time equals the sum of the four sub-metrics */, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id LEFT JOIN user_accounts AS ua ON pr.author_id = ua.account_id LEFT JOIN users AS u ON ua.user_id = u.id LEFT JOIN team_users AS tu ON u.id = tu.user_id LEFT JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.merged_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3, 4, 5, 6, 7, 8) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) THEN deploy_time END) AS \"Team1: Avg Deploy Time(h)\", AVG(CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) THEN deploy_time END) AS \"Team2: Avg Deploy Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Deploy Time(h)", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 139, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "7. PRs Merged w/o Review", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Team1: PRs Merged w/o Review" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Team2: PRs Merged w/o Review" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 66 + }, + "id": 108, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _merged_prs AS (SELECT DISTINCT pr.id, pr.url, pr.created_date, pr.merged_date, pr.author_id, u.id AS user_id, u.name AS user_name, t.id AS team_id, t.name AS team FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' JOIN user_accounts AS ua ON pr.author_id = ua.account_id JOIN users AS u ON ua.user_id = u.id JOIN team_users AS tu ON u.id = tu.user_id JOIN teams AS t ON tu.team_id = t.id WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL ORDER BY 1 NULLS FIRST) SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN ('${team1:csv}' = '' OR team_id::text = ANY(ARRAY[${team1:singlequote}]::text[])) AND NOT id IN (SELECT pull_request_id FROM pull_request_comments) THEN id ELSE NULL END) AS \"Team1: PRs Merged w/o Review\", COUNT(DISTINCT CASE WHEN ('${team2:csv}' = '' OR team_id::text = ANY(ARRAY[${team2:singlequote}]::text[])) AND NOT id IN (SELECT pull_request_id FROM pull_request_comments) THEN id ELSE NULL END) AS \"Team2: PRs Merged w/o Review\" FROM _merged_prs GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Merged w/o Review", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 144, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM teams", + "hide": 0, + "includeAll": false, + "label": "Team1", + "multi": false, + "name": "team1", + "options": [], + "query": "SELECT name || '--' || id AS text FROM teams", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM teams", + "hide": 0, + "includeAll": false, + "label": "Team2", + "multi": false, + "name": "team2", + "options": [], + "query": "SELECT name || '--' || id AS text FROM teams", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT priority FROM issues WHERE priority <> ''", + "hide": 0, + "includeAll": true, + "label": "P0+P1 Priority", + "multi": true, + "name": "priority", + "options": [], + "query": "SELECT DISTINCT priority FROM issues WHERE priority <> ''", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Month", + "value": "DAYOFMONTH" + }, + "description": "", + "hide": 0, + "includeAll": false, + "label": "Time Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Engineering Throughput and Cycle Time - Team View", + "uid": "nJ1ijje7k-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json new file mode 100644 index 00000000000..174ad11e25e --- /dev/null +++ b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json @@ -0,0 +1,1964 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 111, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the engineering throughput and and cycle time, which helps to identify productivity and bottlenecks of the development process.\n- Data Sources Required:\n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira) ([Scope Config](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) required to tell DevLake what the story_points field is)", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 89, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Throughput/Quality", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Request Count", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT pr.id) AS \"PR: Opened\", COUNT(DISTINCT CASE WHEN NOT pr.merged_date IS NULL THEN id ELSE NULL END) AS \"PR: Merged\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Opened/Merged", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Issues Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Issues Completed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 74, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT i.id) AS \"Issues Opened\", COUNT(DISTINCT CASE WHEN i.status = 'DONE' THEN i.id ELSE NULL END) AS \"Issues Completed\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Opened/Completed", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Story Points", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 98, + "links": [ + { + "targetBlank": true, + "title": "Requirement Granularity", + "url": "https://devlake.apache.org/docs/Metrics/RequirementGranularity" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.resolution_date AS DATE)) - 1) END + 1) AS time, SUM(CASE WHEN i.status = 'DONE' THEN i.story_point ELSE 0 END) AS \"Story Points Completed\" FROM (SELECT DISTINCT i.id, i.resolution_date, i.status, i.story_point FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE $__timeFilter(i.resolution_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))) AS i GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Issue Story Points Completed", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "PR Review Depth", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 99, + "links": [ + { + "targetBlank": true, + "title": "PR Review Depth", + "url": "https://devlake.apache.org/docs/Metrics/PRReviewDepth" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT prc.id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS \"PR Review Depth\" FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Review Depth", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Bug Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 100, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT i.id) AS \"P0/P1 Bugs\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id JOIN project_mapping AS pm ON b.id = pm.row_id WHERE $__timeFilter(i.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND i.type = 'BUG' AND ('${priority:csv}' = '' OR i.priority::text = ANY(ARRAY[${priority:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "P0/P1 Bugs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "PR Merged Size", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 101, + "links": [ + { + "targetBlank": true, + "title": "PR Size", + "url": "https://devlake.apache.org/docs/Metrics/PRSize" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _pr_commits_data AS (SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, pr.id AS pr_id, pr.merge_commit_sha, SUM(c.additions) + SUM(c.deletions) AS loc FROM pull_requests AS pr LEFT JOIN commits AS c ON pr.merge_commit_sha = c.sha JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND pr.status = 'MERGED' GROUP BY 1, 2, 3) SELECT time, CAST(SUM(loc) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr_id), 0) AS \"PR Merged Size\" FROM _pr_commits_data GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Merged Size", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 108, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, SUM(CASE WHEN NOT pr.id IN (SELECT pull_request_id FROM pull_request_comments) THEN 1 ELSE 0 END) AS \"PRs Merged w/o Review\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PRs Merged w/o Review", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 103, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Cycle Time", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 109, + "links": [ + { + "targetBlank": true, + "title": "PR Cycle Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCycleTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_cycle_time AS NUMERIC) / NULLIF(60, 0), 0) AS cycle_time /* convert null to 0 if a PR has no cycle_time to make sure cycle_time equals the sum of the four metrics below */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.merged_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(cycle_time) AS \"PR Cycle Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Cycle Time [Merged PRs]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 120, + "links": [ + { + "targetBlank": true, + "title": "PR Coding Time", + "url": "https://devlake.apache.org/docs/Metrics/PRCodingTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_coding_time AS NUMERIC) / NULLIF(60, 0), 0) AS coding_time /* convert null to 0 if a PR has no coding_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.merged_date) AND pr.merged_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(coding_time) AS \"Coding Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Coding Time [Merged PRs]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 33 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "PR Pickup Time", + "url": "https://devlake.apache.org/docs/Metrics/PRPickupTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_pickup_time AS NUMERIC) / NULLIF(60, 0), 0) AS pickup_time /* convert null to 0 if a PR has no pickup_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.merged_date) AND pr.merged_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(pickup_time) AS \"Pickup Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Pickup Time", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 33 + }, + "id": 118, + "links": [ + { + "targetBlank": true, + "title": "PR Review Time", + "url": "https://devlake.apache.org/docs/Metrics/PRReviewTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_review_time AS NUMERIC) / NULLIF(60, 0), 0) AS review_time /* convert null to 0 if a PR has no review_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND pr.merged_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(review_time) AS \"Review Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Review Time [Merged PRs]", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Opened" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 33 + }, + "id": 119, + "links": [ + { + "targetBlank": true, + "title": "PR Deploy Time", + "url": "https://devlake.apache.org/docs/Metrics/PRDeployTime" + } + ], + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT pr.id, pr.merged_date AS pr_merged_date, COALESCE(CAST(prm.pr_deploy_time AS NUMERIC) / NULLIF(60, 0), 0) AS deploy_time /* convert null to 0 if a PR has no deploy_time to make sure cycle_time equals the sum of the four sub-metrics */ FROM pull_requests AS pr LEFT JOIN project_pr_metrics AS prm ON pr.id = prm.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.merged_date) AND pr.merged_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) GROUP BY 1, 2, 3) SELECT CAST(pr_merged_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr_merged_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) END + 1) AS time, AVG(deploy_time) AS \"PR Deploy Time(h)\" FROM _prs GROUP BY 1 ORDER BY 1 NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "PR Deploy Time [Merged PRs]", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 113, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard", + "Highlights" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT priority FROM issues WHERE priority <> ''", + "hide": 0, + "includeAll": true, + "label": "P0+P1 Priority", + "multi": true, + "name": "priority", + "options": [], + "query": "SELECT DISTINCT priority FROM issues WHERE priority <> ''", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Month", + "value": "DAYOFMONTH" + }, + "description": "", + "hide": 0, + "includeAll": false, + "label": "Time Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Engineering Throughput and Cycle Time", + "uid": "Jaaimc67k-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/git-hub.json b/grafana/dashboards/postgresql/git-hub.json new file mode 100644 index 00000000000..48567fd8daf --- /dev/null +++ b/grafana/dashboards/postgresql/git-hub.json @@ -0,0 +1,3658 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 35, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 99, + "links": [ + { + "targetBlank": true, + "title": "GitHub", + "url": "https://devlake.apache.org/docs/Plugins/github" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the basic Git and Code Review metrics from GitHub.\n- Data Source Required: GitHub", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 89, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. User Requirements (Issues)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 4 + }, + "id": 62, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.1 Number of New Issues", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Issue Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 7, + "y": 4 + }, + "id": 74, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT i.id) AS issue_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and i.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, issue_count AS \"Issue Count\" FROM _issues ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1.2 Number of New Issues", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 87, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. How issues are handled?", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 11 + }, + "id": 43, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND i.status = 'DONE'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.1 Number of Closed Issues", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "closed_issue_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 7, + "y": 11 + }, + "id": 111, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status <> 'DONE' THEN i.id ELSE NULL END) AS open_issue_count, COUNT(DISTINCT CASE WHEN status = 'DONE' THEN i.id ELSE NULL END) AS closed_issue_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and i.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, \"open_issue_count\", closed_issue_count FROM _issues ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.2 Number of open and closed issues by Issue creation time", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 14 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 0, + "y": 17 + }, + "id": 64, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS issue_lead_time_in_days FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.resolution_date) AND i.resolution_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND i.status = 'DONE'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.3 Mean Issue Lead Time by issue resolution time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 17, + "x": 7, + "y": 17 + }, + "id": 75, + "links": [ + { + "targetBlank": true, + "title": "Requirement Lead Time", + "url": "https://devlake.apache.org/docs/Metrics/RequirementLeadTime" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _issues AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(i.resolution_date AS DATE)) - 1) END + 1) AS time, AVG(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS issue_lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND i.status = 'DONE' AND $__timeFilter(i.resolution_date) AND i.resolution_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, issue_lead_time AS \"Mean Issue Lead Time in Days\" FROM _issues ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.4 Mean issue lead time by issue resolution time", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Queue Time", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 93, + "links": [], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Get the queue time of all outstanding bugs */ WITH _outstanding_issues AS (SELECT DISTINCT b.name AS repo_name, i.issue_key, i.title, i.created_date, CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS queue_time_in_days, b.url || '/' || i.issue_key AS url FROM issues AS i LEFT JOIN board_issues AS bi ON i.id = bi.issue_id LEFT JOIN boards AS b ON bi.board_id = b.id WHERE ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND i.status <> 'DONE') SELECT '#' || issue_key AS issue_key, title, url, queue_time_in_days FROM _outstanding_issues ORDER BY 4 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.5 Top 20 Queue Time [All Open Issues]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "issue_key" + }, + "properties": [ + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 718 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "created_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 149 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "queue_time_in_days" + }, + "properties": [ + { + "id": "custom.width", + "value": 155 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "repo_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 256 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields.url}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 92, + "links": [], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "/* Get the queue time of all outstanding bugs */ SELECT b.name AS repo_name, i.issue_key AS issue_key, i.title, i.created_date, CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS queue_time_in_days, b.url || '/' || i.issue_key AS url FROM issues AS i LEFT JOIN board_issues AS bi ON i.id = bi.issue_id LEFT JOIN boards AS b ON bi.board_id = b.id WHERE ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(i.created_date) AND i.status <> 'DONE' ORDER BY queue_time_in_days DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "2.6 List of Outstanding Issues Order By Queue Time [All Open Issues]", + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 83, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "3. Contribution (PRs)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 68, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.1 Number of New Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pull Request Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 36 + }, + "id": 77, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT id) AS pr_count FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND $__timeFilter(created_date) /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, pr_count AS \"Pull Request Count\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.2 Number of New Pull Requests", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Merged PR Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 54, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 59, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT author_name, COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'MERGED' GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "3.3 Top 20 Contributors By Merged PRs", + "type": "barchart" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 85, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "4. How PRs are handled?", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.05 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 49 + }, + "id": 66, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.1 Ratio of unmerged PRs of All Closed PRs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "PR: Open" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed without merging" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR: Closed and merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 49 + }, + "id": 79, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN status = 'OPEN' THEN id ELSE NULL END) AS open, COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS closed, COUNT(DISTINCT CASE WHEN status = 'MERGED' THEN id ELSE NULL END) AS merged FROM pull_requests WHERE $__timeFilter(created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, open AS \"PR: Open\", closed AS \"PR: Closed without merging\", merged AS \"PR: Closed and merged\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.2 Pull Request Status Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 55 + }, + "id": 80, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND pr.status = 'CLOSED'", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.3 Number of PRs Closed without Merging", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Ratio", + "axisPlacement": "auto", + "barAlignment": 1, + "drawStyle": "line", + "fillOpacity": 1, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 15, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 55 + }, + "id": 81, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "/* The PR/MR statuses are standardized to DevLake's statuses 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.4 Ratio of unmerged PRs of All Closed PRs", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 61 + }, + "id": 72, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.5 Mean Time to Merge of Pull Requests", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 61 + }, + "id": 95, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_merge FROM pull_requests WHERE $__timeFilter(created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, time_to_merge AS \"Time to Merge\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.6 Mean Time to Merge of Pull Requests in Days", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 67 + }, + "id": 96, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (closed_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_close FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND status IN ('CLOSED', 'MERGED')", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.7 Mean Time to Close of Pull Requests in Days", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 67 + }, + "id": 97, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (closed_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS time_to_close FROM pull_requests WHERE $__timeFilter(created_date) AND /* Enable the following condition to remove the month with incomplete data */ /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND status IN ('CLOSED', 'MERGED') GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, time_to_close AS \"Time to Close\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "4.8 Mean Time to Close of Pull Requests in Days", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 73 + }, + "id": 102, + "panels": [], + "title": "5. CI/CD Metrics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of workflow runs executed in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 74 + }, + "id": 103, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND result = 'SUCCESS' AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.1 Total Number of Successful workflow runs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful workflow runs / Number of total workflow runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 74 + }, + "id": 104, + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(1.0 * COUNT(CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.2 Mean Workflow Runs Success Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "successful_workflow_run_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "failed_workflow_run_count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 109, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.25, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _workflow_runs AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, COUNT(DISTINCT CASE WHEN result = 'SUCCESS' THEN id ELSE NULL END) AS successful_workflow_run_count, COUNT(DISTINCT CASE WHEN result <> 'SUCCESS' THEN id ELSE NULL END) AS failed_workflow_run_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, \"successful_workflow_run_count\", failed_workflow_run_count FROM _workflow_runs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.3 Number of successful and failed workflow runs", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful workflow runs / Number of total workflow runs", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "unit": "none" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "build_count", + "ABORT" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "SUCCESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "FAILURE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ABORT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "rgba(205, 204, 206, 1)", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 80 + }, + "id": 105, + "links": [ + { + "targetBlank": true, + "title": "Build Count", + "url": "https://devlake.apache.org/docs/Metrics/BuildCount" + } + ], + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN result = '' THEN 'Unknown' ELSE result END AS result, COUNT(DISTINCT id) AS build_count FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY 1 ORDER BY 2 DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.4 Total Workflow Runs Result Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Mean Workflow Runs success rate over time.\n2. The workflow runs being calculated are filtered by \"workflow runs starting time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Rate(%)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.9 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Workflow Runs Success Rate" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 80 + }, + "id": 108, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Build Success Rate", + "url": "https://devlake.apache.org/docs/Metrics/BuildSuccessRate" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _build_success_rate AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, result, id FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY \"time\", \"result\", id) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN TO_CHAR(CAST(time AS TIMESTAMP), 'MM/YYYY') ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, CAST(1.0 * SUM(CASE WHEN result = 'SUCCESS' THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Workflow Runs Success Rate\" FROM _build_success_rate GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.5 Workflow Runs Success Rate", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of successful workflow runs / Number of total workflow runs", + "fieldConfig": { + "defaults": { + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-orange", + "value": null + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 86 + }, + "id": 106, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(duration_sec AS NUMERIC) / NULLIF(60, 0)) AS duration_in_minutes FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* the following condition will remove the month with incomplete data */ finished_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH'", + "refId": "A", + "select": [ + [ + { + "params": [ + "project_id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "gitlab_commits", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.6 Mean Workflow Runs Duration", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + }, + { + "color": "red", + "value": 60 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_duration_sec" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 6, + "y": 86 + }, + "id": 110, + "links": [ + { + "targetBlank": true, + "title": "Build Duration", + "url": "https://devlake.apache.org/docs/Metrics/BuildDuration" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.25, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _builds AS (SELECT CAST(finished_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(finished_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) END + 1) AS time, AVG(duration_sec) AS mean_duration_sec FROM cicd_pipelines WHERE $__timeFilter(finished_date) AND id::text LIKE '%github%' AND ('${repo_id:csv}' = '' OR cicd_scope_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) /* Enable the following condition to remove the month with incomplete data */ /* and finished_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ GROUP BY 1) SELECT CASE WHEN '$interval' = 'DAYOFMONTH' THEN (TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY'))::TEXT ELSE 'W' || TO_CHAR(CAST(time AS TIMESTAMP), 'IW YYYY') END AS _time, mean_duration_sec FROM _builds ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "5.7 Mean Workflow Runs Duration", + "type": "barchart" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 101, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'github%'", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM repos WHERE id::text LIKE 'github%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "Month", + "value": "DAYOFMONTH" + }, + "hide": 0, + "includeAll": false, + "label": "Time Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "Month", + "value": "DAYOFMONTH" + }, + { + "selected": false, + "text": "Week", + "value": "WEEKDAY" + } + ], + "query": "Month : DAYOFMONTH, Week : WEEKDAY", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "GitHub", + "uid": "KXWvOFQnz-pg", + "version": 25, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/github-copilot-adoption-by-team-and-user.json b/grafana/dashboards/postgresql/github-copilot-adoption-by-team-and-user.json new file mode 100644 index 00000000000..f2707f6c8f6 --- /dev/null +++ b/grafana/dashboards/postgresql/github-copilot-adoption-by-team-and-user.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "postgresql", + "description": "Per-team and per-user Copilot activity in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.team_slug as \"Team\", udm.user_login as \"User\", SUM(udm.user_initiated_interaction_count) as \"Interactions\", SUM(udm.code_generation_activity_count) as \"Suggestions\", SUM(udm.code_acceptance_activity_count) as \"Acceptances\", ROUND(SUM(udm.code_acceptance_activity_count) * 100.0 / NULLIF(SUM(udm.code_generation_activity_count), 0), 1) as \"Acceptance Rate %\" FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.team_slug, udm.user_login HAVING SUM(udm.code_generation_activity_count) > 0 ORDER BY \"Suggestions\" DESC LIMIT 50", + "refId": "A" + } + ], + "title": "Team and User Breakdown", + "type": "table" + }, + { + "datasource": "postgresql", + "description": "Weekly suggestions by team over the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT DATE(DATE_SUB(ut.day, INTERVAL WEEKDAY(ut.day) DAY)) as time, ut.team_slug as metric, SUM(udm.code_generation_activity_count) as value FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY time, ut.team_slug ORDER BY time", + "refId": "A" + } + ], + "title": "Top Teams by Weekly Suggestions", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "copilot", + "team", + "gh-copilot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "postgresql", + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "postgresql", + "definition": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Copilot Adoption by Team and User", + "uid": "github-copilot-adoption-by-team-and-user", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/postgresql/github-copilot-adoption-by-team.json b/grafana/dashboards/postgresql/github-copilot-adoption-by-team.json new file mode 100644 index 00000000000..2d47835f852 --- /dev/null +++ b/grafana/dashboards/postgresql/github-copilot-adoption-by-team.json @@ -0,0 +1,330 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": "postgresql", + "description": "Distinct teams with Copilot activity in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT team_slug) as \"Active Teams\" FROM _tool_copilot_user_teams WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Active Teams", + "type": "stat" + }, + { + "datasource": "postgresql", + "description": "Distinct users mapped to teams in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) as \"Active Users\" FROM _tool_copilot_user_teams WHERE connection_id = ${connection_id} AND scope_id = '${scope_id}' AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Active Users in Teams", + "type": "stat" + }, + { + "datasource": "postgresql", + "description": "Teams ranked by Copilot suggestions generated in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.team_slug as \"Team\", COUNT(DISTINCT ut.user_id) as \"Users\", SUM(udm.code_generation_activity_count) as \"Suggestions\", SUM(udm.code_acceptance_activity_count) as \"Acceptances\", ROUND(SUM(udm.code_acceptance_activity_count) * 100.0 / NULLIF(SUM(udm.code_generation_activity_count), 0), 1) as \"Acceptance Rate %\" FROM _tool_copilot_user_teams ut JOIN _tool_copilot_user_daily_metrics udm ON ut.connection_id = udm.connection_id AND ut.scope_id = udm.scope_id AND ut.day = udm.day AND ut.user_id = udm.user_id WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.team_slug HAVING SUM(udm.code_generation_activity_count) > 0 ORDER BY \"Suggestions\" DESC LIMIT 15", + "refId": "A" + } + ], + "title": "Top Teams by Suggestions", + "type": "table" + }, + { + "datasource": "postgresql", + "description": "Daily active users by team over the selected time range", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": "postgresql", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ut.day as time, ut.team_slug as metric, COUNT(DISTINCT ut.user_id) as value FROM _tool_copilot_user_teams ut WHERE ut.connection_id = ${connection_id} AND ut.scope_id = '${scope_id}' AND $__timeFilter(ut.day) GROUP BY ut.day, ut.team_slug ORDER BY ut.day", + "refId": "A" + } + ], + "title": "Daily Active Team Users", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "copilot", + "team", + "gh-copilot" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "postgresql", + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": "postgresql", + "definition": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id as scope_id FROM _tool_copilot_scopes WHERE connection_id = ${connection_id} ORDER BY 1", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "GitHub Copilot Adoption by Team", + "uid": "github-copilot-adoption-by-team", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/postgresql/github-copilot-adoption.json b/grafana/dashboards/postgresql/github-copilot-adoption.json new file mode 100644 index 00000000000..7a4eee270a0 --- /dev/null +++ b/grafana/dashboards/postgresql/github-copilot-adoption.json @@ -0,0 +1,2322 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Latest daily active users count", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT daily_active_users AS \"Daily Active Users\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY day DESC NULLS LAST LIMIT 1", + "refId": "A" + } + ], + "title": "Daily Active Users", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code acceptance rate across the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 20 + }, + { + "color": "green", + "value": 40 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 2, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Code Acceptance Rate", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total lines of code added by Copilot in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(loc_added_sum) AS \"Lines Added\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Lines of Code Added", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total pull requests created by Copilot in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(pr_total_created_by_copilot) AS \"PRs by Copilot\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "PRs Created by Copilot", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Latest weekly active users count", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_login) AS \"Weekly Active Users\" FROM _tool_copilot_user_daily_metrics WHERE day >= CURRENT_DATE - INTERVAL '7 DAY' AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Weekly Active Users (WAU)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Latest monthly active users count", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_login) AS \"Monthly Active Users\" FROM _tool_copilot_user_daily_metrics WHERE day >= CURRENT_DATE - INTERVAL '28 DAY' AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Monthly Active Users (MAU)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total pull requests reviewed by Copilot in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(pr_total_reviewed_by_copilot) AS \"PRs Reviewed by Copilot\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "PRs Reviewed by Copilot", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total user-initiated interactions in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 18, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(user_initiated_interaction_count) AS \"User-Initiated Interactions\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "User-Initiated Interactions", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily and monthly active users over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Users", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, daily_active_users AS \"Daily Active\", monthly_active_users AS \"Monthly Active\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Active Users Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code suggestions generated vs accepted over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Count", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, code_generation_activity_count AS \"Suggestions\", code_acceptance_activity_count AS \"Acceptances\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Code Suggestions & Acceptances", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Lines of code suggested to add versus lines added over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Lines of Code", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, loc_suggested_to_add_sum AS \"LOC Suggested\", loc_added_sum AS \"LOC Added\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "LOC Suggested vs LOC Added Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pull request activity over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Pull Requests", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, pr_total_created AS \"PRs Created\", pr_total_created_by_copilot AS \"PRs Created by Copilot\", pr_total_reviewed AS \"PRs Reviewed\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "PR Activity Over Time", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly user-initiated interactions by top Copilot features", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Interactions", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "normal", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CAST(day - (EXTRACT(ISODOW FROM day) - 1) * INTERVAL '1 day' AS DATE) AS time, CASE WHEN feature IN (SELECT feature FROM (SELECT feature FROM _tool_copilot_metrics_by_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"feature\" HAVING SUM(user_initiated_interaction_count) > 0 ORDER BY SUM(user_initiated_interaction_count) DESC NULLS LAST LIMIT 4) AS top_features) THEN REPLACE(REPLACE(feature, 'chat_panel_', ''), '_mode', '') ELSE 'Other' END AS metric, SUM(user_initiated_interaction_count) AS value FROM _tool_copilot_metrics_by_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"time\", \"metric\" ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Feature Mix Over Time", + "type": "timeseries", + "transformations": [ + { + "id": "prepareTimeSeries", + "options": { + "format": "many" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^value (.+)$", + "renamePattern": "$1" + } + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly user-initiated interactions by top IDEs (Top 5 + Other)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Weekly Interactions", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 0, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "mode": "normal", + "group": "A" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "stacking": { + "mode": "normal", + "group": "A" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CAST(day - (EXTRACT(ISODOW FROM day) - 1) * INTERVAL '1 day' AS DATE) AS time, CASE WHEN ide IN (SELECT ide FROM (SELECT ide FROM _tool_copilot_metrics_by_ide WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"ide\" HAVING SUM(user_initiated_interaction_count) > 0 ORDER BY SUM(user_initiated_interaction_count) DESC NULLS LAST LIMIT 4) AS top_ides) THEN ide ELSE 'Other' END AS metric, SUM(user_initiated_interaction_count) AS value FROM _tool_copilot_metrics_by_ide WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"time\", \"metric\" ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "IDE Adoption Over Time", + "type": "timeseries", + "transformations": [ + { + "id": "prepareTimeSeries", + "options": { + "format": "many" + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "^value (.+)$", + "renamePattern": "$1" + } + } + ] + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Top languages ranked by code acceptance rate", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 9, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT language AS \"Language\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_metrics_by_language_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"language\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) DESC NULLS LAST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 15", + "refId": "A" + } + ], + "title": "Top Languages by Acceptance Rate", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Top models ranked by code acceptance rate", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT model AS \"Model\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_metrics_by_model_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"model\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) DESC NULLS LAST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 10", + "refId": "A" + } + ], + "title": "Top Models by Acceptance Rate", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Matrix-style breakdown of suggestions and acceptances by language and feature", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT language AS \"Language\", feature AS \"Feature\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\" FROM _tool_copilot_metrics_by_language_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"language\", \"feature\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY language NULLS FIRST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 100", + "refId": "A" + } + ], + "title": "Feature x Language Matrix", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Count of distinct users active in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 56 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_login) AS \"Unique Users\" FROM _tool_copilot_user_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Unique Users (Period)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Users who have used Copilot agent mode", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 56 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_login) AS \"Agent Users\" FROM _tool_copilot_user_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND used_agent = 1", + "refId": "A" + } + ], + "title": "Agent Mode Adopters", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Users who have used Copilot chat", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "orange", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 56 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_login) AS \"Chat Users\" FROM _tool_copilot_user_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND used_chat = 1", + "refId": "A" + } + ], + "title": "Chat Adopters", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage of seats with recorded activity", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 75 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 56 + }, + "id": 14, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CAST(COUNT(CASE WHEN NOT last_activity_at IS NULL THEN 1 END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(*), 0), 0) AS \"Utilization %\" FROM _tool_copilot_seats WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}')", + "refId": "A" + } + ], + "title": "Seat Utilization", + "type": "gauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 22, + "panels": [], + "title": "Quality & Efficiency Ratios", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily acceptance rate trend computed as acceptances divided by suggestions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "max": 1, + "min": 0, + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 61 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, CAST(code_acceptance_activity_count AS NUMERIC) / NULLIF(NULLIF(code_generation_activity_count, 0), 0) AS \"Acceptance Rate\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Overall Acceptance Rate Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily LOC yield trend computed as added LOC divided by suggested LOC", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, ROUND(CAST(loc_added_sum AS NUMERIC) / NULLIF(NULLIF(loc_suggested_to_add_sum, 0), 0), 2) AS \"LOC Yield\" FROM _tool_copilot_enterprise_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "LOC Yield Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Acceptance ratio by Copilot feature", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 69 + }, + "id": 25, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT feature AS \"Feature\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_metrics_by_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"feature\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) DESC NULLS LAST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "Acceptance Rate by Feature", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Acceptance ratio by programming language", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 69 + }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT language AS \"Language\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_metrics_by_language_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"language\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) DESC NULLS LAST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "Acceptance Rate by Language", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Acceptance ratio by model", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 69 + }, + "id": 27, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT model AS \"Model\", SUM(code_generation_activity_count) AS \"Suggestions\", SUM(code_acceptance_activity_count) AS \"Acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"Acceptance Rate %\" FROM _tool_copilot_metrics_by_model_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"model\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) DESC NULLS LAST, SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 20", + "refId": "A" + } + ], + "title": "Acceptance Rate by Model", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 28, + "panels": [], + "title": "User Behavior", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Top users ranked by code generation activity in the selected time range", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 29, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT user_login AS \"user_login\", SUM(code_generation_activity_count) AS \"suggestions\", SUM(code_acceptance_activity_count) AS \"acceptances\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) AS \"acceptance %\" FROM _tool_copilot_user_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"user_login\" HAVING SUM(code_generation_activity_count) > 0 ORDER BY SUM(code_generation_activity_count) DESC NULLS LAST LIMIT 25", + "refId": "A" + } + ], + "title": "Top Users by Code Generations", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily trend of distinct users engaging with agent mode versus chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Users", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT day AS time, COUNT(DISTINCT CASE WHEN used_agent = 1 THEN user_login END) AS \"Agent Users\", COUNT(DISTINCT CASE WHEN used_chat = 1 THEN user_login END) AS \"Chat Users\" FROM _tool_copilot_user_daily_metrics WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY \"day\" ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "Agent Users vs Chat Users Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "First-seen daily trend of users becoming active for the first time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Users", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 86 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT first_day AS time, COUNT(*) AS \"New Active Users\" FROM (SELECT user_login, MIN(day) AS first_day FROM _tool_copilot_user_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY user_login) AS first_seen WHERE $__timeFilter(first_day) GROUP BY first_day ORDER BY 1 NULLS FIRST", + "refId": "A" + } + ], + "title": "New Active Users per Day", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 94 + }, + "id": 32, + "panels": [], + "title": "Seat Effectiveness", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of seat activity by last recorded editor", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 33, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(NULLIF(last_activity_editor, ''), 'unknown') AS \"Editor\", COUNT(*) AS \"Seats\" FROM _tool_copilot_seats WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '') GROUP BY COALESCE(NULLIF(last_activity_editor, ''), 'unknown') ORDER BY COUNT(*) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Seat Activity by Editor", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total, active (last 30 days), and inactive seats for the selected scope", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 95 + }, + "id": 34, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"total_seats\", SUM(CASE WHEN NOT last_activity_at IS NULL AND last_activity_at >= CURRENT_TIMESTAMP - INTERVAL '30 DAY' THEN 1 ELSE 0 END) AS \"active_seats\", SUM(CASE WHEN last_activity_at IS NULL OR last_activity_at < CURRENT_TIMESTAMP - INTERVAL '30 DAY' THEN 1 ELSE 0 END) AS \"inactive_seats\" FROM _tool_copilot_seats WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')", + "refId": "A" + } + ], + "title": "Seats Summary", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Seats with no activity or activity older than 30 days", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 95 + }, + "id": 35, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT user_login AS \"user_login\", last_activity_at AS \"last_activity_at\", plan_type AS \"plan_type\" FROM _tool_copilot_seats WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '') AND (last_activity_at IS NULL OR last_activity_at < CURRENT_TIMESTAMP - INTERVAL '30 DAY') ORDER BY last_activity_at IS NULL DESC NULLS LAST, last_activity_at ASC NULLS FIRST LIMIT 100", + "refId": "A" + } + ], + "title": "Inactive Seats", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 36, + "panels": [], + "title": "Diagnostics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Latest available day in enterprise-level Copilot metrics", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 104 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE((EXTRACT(EPOCH FROM (CURRENT_DATE AT TIME ZONE 'UTC' - MAX(day)))/86400), 9999) AS \"Days Since Latest Data\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Data Freshness", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Share of generated suggestions tied to unknown model/feature taxonomy", + "fieldConfig": { + "defaults": { + "mappings": [], + "min": 0, + "max": 100, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 104 + }, + "id": 38, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(CASE WHEN LOWER(COALESCE(model, '')) = 'unknown' OR LOWER(COALESCE(feature, '')) LIKE '%unknown%' THEN code_generation_activity_count ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 2) AS \"Unknown Taxonomy %\" FROM _tool_copilot_metrics_by_model_feature WHERE $__timeFilter(day) AND ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}'", + "refId": "A" + } + ], + "title": "Unknown Taxonomy Share", + "type": "gauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Row counts for core Copilot metrics tables", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 104 + }, + "id": 39, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT * FROM (SELECT '_tool_copilot_enterprise_daily_metrics' AS table_name, COUNT(*) AS row_count FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' UNION ALL SELECT '_tool_copilot_user_daily_metrics' AS table_name, COUNT(*) AS row_count FROM _tool_copilot_user_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' UNION ALL SELECT '_tool_copilot_metrics_by_feature' AS table_name, COUNT(*) AS row_count FROM _tool_copilot_metrics_by_feature WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' UNION ALL SELECT '_tool_copilot_metrics_by_model_feature' AS table_name, COUNT(*) AS row_count FROM _tool_copilot_metrics_by_model_feature WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' UNION ALL SELECT '_tool_copilot_seats' AS table_name, COUNT(*) AS row_count FROM _tool_copilot_seats WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND ('${scope_id}' = '' OR organization = '${scope_id}' OR organization = '')) AS data_volume ORDER BY row_count DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Data Volume by Table", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "copilot", + "devlake" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_enterprise_daily_metrics ORDER BY 1 NULLS FIRST", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC NULLS LAST", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT id AS scope_id FROM _tool_copilot_scopes WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') ORDER BY 1 NULLS FIRST", + "hide": 0, + "includeAll": false, + "label": "Scope ID", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id AS scope_id FROM _tool_copilot_scopes WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') ORDER BY 1 NULLS FIRST", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "GitHub Copilot Adoption", + "uid": "copilot_adoption-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/github-copilot-dora-correlation.json b/grafana/dashboards/postgresql/github-copilot-dora-correlation.json new file mode 100644 index 00000000000..847f4ec7ead --- /dev/null +++ b/grafana/dashboards/postgresql/github-copilot-dora-correlation.json @@ -0,0 +1,2931 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "enable": true, + "hide": false, + "iconColor": "purple", + "name": "Copilot Implementation Date", + "rawQuery": "SELECT\n UNIX_TIMESTAMP(implementation_date) * 1000 AS time,\n 'Copilot Rollout' AS text,\n 'Implementation Date' AS title\nFROM _tool_copilot_scopes\nWHERE id = '${scope_id}'\n AND implementation_date IS NOT NULL", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## GitHub Copilot + DORA Correlation Dashboard\n\n**Purpose**: Analyze the **correlation** between GitHub Copilot adoption levels and engineering productivity metrics.\n\n**How it works**:\n- \ud83d\udcc8 **Primary Analysis**: Continuous correlation showing how DORA metrics trend alongside GitHub Copilot adoption intensity\n- \ud83d\udcca **Adoption Tiers**: Metrics grouped by adoption level (<25%, 25-50%, 50-75%, >75% of seats active)\n- \ud83d\udccd **Optional Milestone**: Configure a rollout date to add annotation markers on charts\n\n**Prerequisites**:\n- Ensure **GitHub Copilot metrics** are being collected via the GitHub Copilot plugin\n- Ensure **PR data** exists in DevLake from GitHub/GitLab plugins\n- For DORA metrics (CFR, MTTR): Configure deployments and incidents\n- For Code Quality: Configure SonarQube integration (optional)\n\n**\u26a0\ufe0f Disclaimer**: Correlation does not imply causation. Other factors may influence the metrics shown.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly GitHub Copilot adoption percentage (active users / total seats) over time. Higher adoption correlates with expected productivity improvements.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Adoption %", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + }, + { + "color": "green", + "value": 50 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, active_users, total_seats, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, ROUND(CAST(AVG(adoption_pct) AS DECIMAL), 1) AS adoption_pct, ROUND(CAST(AVG(active_users) AS DECIMAL), 0) AS avg_active_users, ROUND(CAST(AVG(total_seats) AS DECIMAL), 0) AS avg_total_seats FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day') SELECT week_start AS time, adoption_pct AS \"Adoption %\" FROM _adoption_weekly ORDER BY week_start NULLS FIRST", + "refId": "A" + } + ], + "title": "GitHub Copilot Adoption Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient (r) between GitHub Copilot adoption % and PR cycle time. Negative values indicate inverse correlation (higher adoption \u2192 faster PRs). |r| > 0.7 = strong, 0.3-0.7 = moderate, < 0.3 = weak.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "from": -1, + "result": { + "color": "green", + "index": 0, + "text": "Strong \u2193" + }, + "to": -0.7 + }, + "type": "range" + }, + { + "options": { + "from": -0.7, + "result": { + "color": "yellow", + "index": 1, + "text": "Moderate \u2193" + }, + "to": -0.3 + }, + "type": "range" + }, + { + "options": { + "from": -0.3, + "result": { + "color": "text", + "index": 2, + "text": "Weak" + }, + "to": 0.3 + }, + "type": "range" + }, + { + "options": { + "from": 0.3, + "result": { + "color": "orange", + "index": 3, + "text": "Moderate \u2191" + }, + "to": 0.7 + }, + "type": "range" + }, + { + "options": { + "from": 0.7, + "result": { + "color": "red", + "index": 4, + "text": "Strong \u2191" + }, + "to": 1 + }, + "type": "range" + } + ], + "noValue": "\u26a0\ufe0f Insufficient data (need 4+ weeks)", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none", + "decimals": 2 + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 31 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_metrics_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _correlation_data AS (SELECT aw.week_start, aw.adoption_pct, pm.avg_cycle_time_hours FROM _adoption_weekly AS aw INNER JOIN _pr_metrics_weekly AS pm ON aw.week_start = pm.week_start WHERE NOT aw.adoption_pct IS NULL AND NOT pm.avg_cycle_time_hours IS NULL), _stats AS (SELECT COUNT(*) AS n, AVG(adoption_pct) AS mean_x, AVG(avg_cycle_time_hours) AS mean_y, STDDEV_POP(adoption_pct) AS stddev_x, STDDEV_POP(avg_cycle_time_hours) AS stddev_y FROM _correlation_data), _pearson AS (SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.stddev_x = '0' OR s.stddev_y = '0' THEN 0 ELSE (SELECT CAST(SUM((cd.adoption_pct - s.mean_x) * (cd.avg_cycle_time_hours - s.mean_y)) AS NUMERIC) / NULLIF((s.n * s.stddev_x * s.stddev_y), 0) FROM _correlation_data AS cd) END AS r FROM _stats AS s) SELECT ROUND(r, 2) AS value FROM _pearson", + "refId": "A" + } + ], + "title": "Adoption vs PR Cycle Time (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Current week's GitHub Copilot adoption percentage (active users / total seats)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "No data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 25 + }, + { + "color": "green", + "value": 50 + }, + { + "color": "dark-green", + "value": 75 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS value FROM (SELECT daily_active_users AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY day DESC NULLS LAST LIMIT 1) AS _ent UNION ALL SELECT ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS value FROM (SELECT total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' ORDER BY date DESC NULLS LAST LIMIT 1) AS _org LIMIT 1", + "refId": "A" + } + ], + "title": "Current Adoption %", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 6, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "### Interpreting Correlation (r)\n\n| Value | Meaning |\n|-------|--------|\n| **-1 to -0.7** | \ud83d\udfe2 Strong inverse (higher adoption \u2192 faster PRs) |\n| **-0.7 to -0.3** | \ud83d\udfe1 Moderate inverse |\n| **-0.3 to 0.3** | \u26aa Weak/No correlation |\n| **0.3 to 0.7** | \ud83d\udfe0 Moderate positive |\n| **0.7 to 1** | \ud83d\udd34 Strong positive (unexpected) |\n\n*Note: r < 0 is expected (adoption \u2191 = cycle time \u2193)*", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "\ud83d\udcca How to Read Correlation", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 100, + "panels": [], + "title": "PR Velocity Impact", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage difference in PR cycle time between high adoption weeks (>50%) and low adoption weeks (<50%). Negative values indicate improvement (faster PRs during high adoption).", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "\u26a0\ufe0f Insufficient data", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 16, + "y": 31 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_metrics_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_pr_joined AS (SELECT aw.week_start, aw.adoption_pct, pm.avg_cycle_time_hours, CASE WHEN aw.adoption_pct >= 50 THEN 'high' ELSE 'low' END AS adoption_tier FROM _adoption_weekly AS aw INNER JOIN _pr_metrics_weekly AS pm ON aw.week_start = pm.week_start), _tier_averages AS (SELECT adoption_tier, AVG(avg_cycle_time_hours) AS avg_cycle_time FROM _adoption_pr_joined GROUP BY adoption_tier) SELECT ROUND(CAST(((SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'high') - (SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'low')) AS NUMERIC) / NULLIF(NULLIF((SELECT avg_cycle_time FROM _tier_averages WHERE adoption_tier = 'low'), 0), 0) * 100, 1) AS value", + "refId": "A" + } + ], + "title": "PR Cycle Time Change (High vs Low Adoption)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Dual-axis chart showing weekly GitHub Copilot adoption % (left axis) and PR cycle time in hours (right axis). Look for inverse correlation - adoption up, cycle time down.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Adoption %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "max", + "value": 100 + }, + { + "id": "min", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR Cycle Time (hrs)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "h" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day') SELECT week_start AS time, ROUND(adoption_pct, 1) AS \"Adoption %\" FROM _adoption_weekly ORDER BY week_start NULLS FIRST", + "refId": "Adoption" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _pr_metrics_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT week_start AS time, ROUND(avg_cycle_time_hours, 1) AS \"PR Cycle Time (hrs)\" FROM _pr_metrics_weekly ORDER BY week_start NULLS FIRST", + "refId": "PRTime" + } + ], + "title": "Adoption vs PR Cycle Time", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average PR cycle time by adoption tier. Lower times in higher adoption tiers suggest positive Copilot impact.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "red", + "value": 48 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 12, + "y": 31 + }, + "id": 14, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "transformations": [ + { + "id": "rowsToFields", + "options": { + "labelField": "metric", + "valueField": "value" + } + } + ], + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_metrics_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_pr_joined AS (SELECT aw.adoption_pct, pm.avg_cycle_time_hours, CASE WHEN aw.adoption_pct < 25 THEN '1. <25%' WHEN aw.adoption_pct < 50 THEN '2. 25-50%' WHEN aw.adoption_pct < 75 THEN '3. 50-75%' ELSE '4. >75%' END AS adoption_tier FROM _adoption_weekly AS aw INNER JOIN _pr_metrics_weekly AS pm ON aw.week_start = pm.week_start) SELECT adoption_tier AS metric, ROUND(CAST(AVG(avg_cycle_time_hours) AS DECIMAL), 1) AS value FROM _adoption_pr_joined GROUP BY adoption_tier ORDER BY adoption_tier NULLS FIRST", + "refId": "A" + } + ], + "title": "PR Cycle Time by Adoption Tier", + "type": "bargauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 35 + }, + "id": 15, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "**Cycle Time Components** break down PR lifecycle into three phases:\n\n- **Coding Time**: PR creation \u2192 first commit (development speed)\n- **Pickup Time**: PR creation \u2192 first review (reviewer responsiveness)\n- **Review Time**: First review \u2192 merge (actual review duration)\n\nComparing <50% vs \u226550% adoption periods shows which phase improved most with Copilot. For example, if Coding Time shrinks significantly, Copilot accelerates development.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Understanding Cycle Time Breakdown", + "type": "text" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Scatter plot showing weekly data points. Each point = one week. X-axis = adoption %, Y-axis = PR cycle time. Downward trend line indicates positive Copilot impact.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Adoption %", + "axisPlacement": "auto", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "pointSize": { + "fixed": 10 + }, + "scaleDistribution": { + "type": "linear" + }, + "show": "points" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 11, + "options": { + "dims": { + "x": "adoption_pct" + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "series": [], + "seriesMapping": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_metrics_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT ROUND(aw.adoption_pct, 1) AS adoption_pct, ROUND(pm.avg_cycle_time_hours, 1) AS cycle_time_hours FROM _adoption_weekly AS aw INNER JOIN _pr_metrics_weekly AS pm ON aw.week_start = pm.week_start ORDER BY aw.adoption_pct NULLS FIRST", + "refId": "A" + } + ], + "title": "Adoption vs PR Cycle Time (Scatter)", + "type": "xychart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "PR cycle time breakdown during low adoption periods (<50%). Compare with high adoption to see which components improved.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 35 + }, + "id": 12, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _low_adoption_weeks AS (SELECT week_start FROM _adoption_weekly WHERE adoption_pct < 50) SELECT 'Coding Time' AS component, ROUND(CAST(CAST(AVG(pr_coding_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _low_adoption_weeks) UNION ALL SELECT 'Pickup Time' AS component, ROUND(CAST(CAST(AVG(pr_pickup_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _low_adoption_weeks) UNION ALL SELECT 'Review Time' AS component, ROUND(CAST(CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _low_adoption_weeks)", + "refId": "A" + } + ], + "title": "Cycle Time Components: Low Adoption", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "PR cycle time breakdown during high adoption periods (>=50%). Compare with low adoption to see which components improved.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 35 + }, + "id": 13, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _high_adoption_weeks AS (SELECT week_start FROM _adoption_weekly WHERE adoption_pct >= 50) SELECT 'Coding Time' AS component, ROUND(CAST(CAST(AVG(pr_coding_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _high_adoption_weeks) UNION ALL SELECT 'Pickup Time' AS component, ROUND(CAST(CAST(AVG(pr_pickup_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _high_adoption_weeks) UNION ALL SELECT 'Review Time' AS component, ROUND(CAST(CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS value FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' IN (SELECT week_start FROM _high_adoption_weeks)", + "refId": "A" + } + ], + "title": "Cycle Time Components: High Adoption", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 200, + "panels": [], + "title": "Deployment Frequency Impact", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Dual-axis chart showing GitHub Copilot adoption percentage and deployment frequency over time. Look for correlation patterns - does higher adoption correlate with more frequent deployments?", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Adoption %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deployments" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS deploy_count FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT EXTRACT(EPOCH FROM aw.week_start) AS time_sec, aw.adoption_pct AS \"Adoption %\", COALESCE(dw.deploy_count, 0) AS \"Deployments\" FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start ORDER BY aw.week_start NULLS FIRST", + "refId": "A" + } + ], + "title": "Adoption vs Deployment Frequency", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Deployment frequency grouped by GitHub Copilot adoption tier. Higher adoption should correlate with more frequent deployments if Copilot accelerates development velocity.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "green", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 46 + }, + "id": 32, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "transformations": [ + { + "id": "rowsToFields", + "options": { + "labelField": "Adoption Tier", + "valueField": "Deploys/Week" + } + } + ], + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct, CASE WHEN AVG(adoption_pct) < 25 THEN '1: <25%' WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%' WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%' ELSE '4: >75%' END AS adoption_tier FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS deploy_count FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT aw.adoption_tier AS \"Adoption Tier\", ROUND(CAST(AVG(COALESCE(dw.deploy_count, 0)) AS DECIMAL), 1) AS \"Deploys/Week\" FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start GROUP BY aw.adoption_tier ORDER BY aw.adoption_tier NULLS FIRST", + "refId": "A" + } + ], + "title": "Deployments per Week by Adoption Tier", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient (r) between GitHub Copilot adoption % and weekly deployment count. Positive values indicate higher adoption correlates with more deployments.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.3 + }, + { + "color": "green", + "value": 0.7 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 46 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS deploy_count FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT aw.adoption_pct, COALESCE(dw.deploy_count, 0) AS deploy_count FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start) SELECT ROUND(CAST(CAST((COUNT(*) * SUM(adoption_pct * deploy_count) - SUM(adoption_pct) * SUM(deploy_count)) AS NUMERIC) / NULLIF(NULLIF(SQRT((COUNT(*) * SUM(adoption_pct * adoption_pct) - POWER(SUM(adoption_pct), 2)) * (COUNT(*) * SUM(deploy_count * deploy_count) - POWER(SUM(deploy_count), 2))), 0), 0) AS DECIMAL), 2) AS correlation_r FROM _joined WHERE NOT adoption_pct IS NULL", + "refId": "A" + } + ], + "title": "Adoption vs Deployment Frequency (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage change in deployment frequency between low adoption (<50%) and high adoption (\u226550%) periods.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0 + }, + { + "color": "green", + "value": 20 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 50 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS deploy_count FROM cicd_deployment_commits WHERE result = 'SUCCESS' AND $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_deploy AS (SELECT CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier, COALESCE(dw.deploy_count, 0) AS deploy_count FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start), _tier_avg AS (SELECT tier, AVG(deploy_count) AS avg_deploys FROM _adoption_deploy GROUP BY tier) SELECT ROUND(CAST(((SELECT avg_deploys FROM _tier_avg WHERE tier = 'high') - (SELECT avg_deploys FROM _tier_avg WHERE tier = 'low')) AS NUMERIC) / NULLIF(NULLIF((SELECT avg_deploys FROM _tier_avg WHERE tier = 'low'), 0), 0) * 100, 1) AS change_pct", + "refId": "A" + } + ], + "title": "Deployment Frequency Change (High vs Low Adoption)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 250, + "panels": [], + "title": "Adoption Intensity Analysis", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Comprehensive comparison of all DORA metrics across GitHub Copilot adoption tiers. Lower values (green) are better for cycle time, CFR, and MTTR. Higher values (green) are better for deployment frequency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 72, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _adoption_tiers AS (SELECT week_start, \"adoption_pct\", CASE WHEN adoption_pct < 25 THEN '<25%' WHEN adoption_pct < 50 THEN '25-50%' WHEN adoption_pct < 75 THEN '50-75%' ELSE '>75%' END AS tier FROM _adoption_weekly), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG(CAST(pr_cycle_time AS NUMERIC) / NULLIF(60, 0)) AS cycle_time_hrs FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND NOT pr_merged_date IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _deploy_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS deploy_count, SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys FROM cicd_deployment_commits WHERE $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day'), _cfr_weekly AS (SELECT week_start, ROUND(CAST(failed_deploys * 100.0 AS NUMERIC) / NULLIF(NULLIF(deploy_count, 0), 0), 1) AS cfr_pct FROM _deploy_weekly), _mttr_weekly AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG((EXTRACT(EPOCH FROM (resolution_date - created_date))/3600)) AS mttr_hours FROM issues WHERE type = 'INCIDENT' AND NOT resolution_date IS NULL AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day'), _tier_metrics AS (SELECT at.tier, ROUND(CAST(AVG(pw.cycle_time_hrs) AS DECIMAL), 1) AS pr_cycle_time, ROUND(CAST(AVG(dw.deploy_count) AS DECIMAL), 1) AS deploy_freq, ROUND(CAST(AVG(cfr.cfr_pct) AS DECIMAL), 1) AS cfr_pct, ROUND(CAST(AVG(mttr.mttr_hours) AS DECIMAL), 1) AS mttr_hours FROM _adoption_tiers AS at LEFT JOIN _pr_weekly AS pw ON at.week_start = pw.week_start LEFT JOIN _deploy_weekly AS dw ON at.week_start = dw.week_start LEFT JOIN _cfr_weekly AS cfr ON at.week_start = cfr.week_start LEFT JOIN _mttr_weekly AS mttr ON at.week_start = mttr.week_start GROUP BY at.tier) SELECT tier AS \"Adoption Tier\", COALESCE(pr_cycle_time, 0) AS \"PR Cycle Time (hrs)\", COALESCE(deploy_freq, 0) AS \"Deploys/Week\", COALESCE(cfr_pct, 0) AS \"CFR %\", COALESCE(mttr_hours, 0) AS \"MTTR (hours)\" FROM _tier_metrics ORDER BY CASE tier WHEN '<25%' THEN 1 WHEN '25-50%' THEN 2 WHEN '50-75%' THEN 3 WHEN '>75%' THEN 4 END NULLS FIRST", + "refId": "A" + } + ], + "title": "DORA Metrics by Adoption Tier", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 300, + "panels": [], + "title": "Change Failure Rate Impact", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Dual-axis chart showing GitHub Copilot adoption percentage and Change Failure Rate over time. CFR should decrease (improve) as adoption increases if Copilot improves code quality.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Adoption %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CFR %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS total_deploys, SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys FROM cicd_deployment_commits WHERE $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT EXTRACT(EPOCH FROM aw.week_start) AS time_sec, aw.adoption_pct AS \"Adoption %\", ROUND(CAST(dw.failed_deploys * 100.0 AS NUMERIC) / NULLIF(NULLIF(dw.total_deploys, 0), 0), 1) AS \"CFR %\" FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start ORDER BY aw.week_start NULLS FIRST", + "refId": "A" + } + ], + "title": "Adoption vs Change Failure Rate", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Change Failure Rate grouped by GitHub Copilot adoption tier. Lower CFR at higher adoption indicates Copilot improves code quality and reduces failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 15 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 55 + }, + "id": 42, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "transformations": [ + { + "id": "rowsToFields", + "options": { + "labelField": "Adoption Tier", + "valueField": "CFR %" + } + } + ], + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct, CASE WHEN AVG(adoption_pct) < 25 THEN '1: <25%' WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%' WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%' ELSE '4: >75%' END AS adoption_tier FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS total_deploys, SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys FROM cicd_deployment_commits WHERE $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day') SELECT aw.adoption_tier AS \"Adoption Tier\", ROUND(CAST(AVG(CAST(dw.failed_deploys * 100.0 AS NUMERIC) / NULLIF(NULLIF(dw.total_deploys, 0), 0)) AS DECIMAL), 1) AS \"CFR %\" FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start GROUP BY aw.adoption_tier ORDER BY aw.adoption_tier NULLS FIRST", + "refId": "A" + } + ], + "title": "CFR by Adoption Tier", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient (r) between GitHub Copilot adoption % and Change Failure Rate. NEGATIVE values are good - they indicate higher adoption correlates with lower failure rates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -0.3 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 55 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS total_deploys, SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys FROM cicd_deployment_commits WHERE $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT aw.adoption_pct, (CAST(dw.failed_deploys * 100.0 AS NUMERIC) / NULLIF(NULLIF(dw.total_deploys, 0), 0)) AS cfr_pct FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start WHERE dw.total_deploys > 0) SELECT ROUND(CAST(CAST((COUNT(*) * SUM(adoption_pct * cfr_pct) - SUM(adoption_pct) * SUM(cfr_pct)) AS NUMERIC) / NULLIF(NULLIF(SQRT((COUNT(*) * SUM(adoption_pct * adoption_pct) - POWER(SUM(adoption_pct), 2)) * (COUNT(*) * SUM(cfr_pct * cfr_pct) - POWER(SUM(cfr_pct), 2))), 0), 0) AS DECIMAL), 2) AS correlation_r FROM _joined", + "refId": "A" + } + ], + "title": "Adoption vs CFR (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage change in Change Failure Rate between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate improvement.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -10 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 59 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _deployments_weekly AS (SELECT CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(*) AS total_deploys, SUM(CASE WHEN result = 'FAILURE' THEN 1 ELSE 0 END) AS failed_deploys FROM cicd_deployment_commits WHERE $__timeFilter(finished_date) GROUP BY CAST(finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(finished_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_cfr AS (SELECT CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier, (CAST(dw.failed_deploys * 100.0 AS NUMERIC) / NULLIF(NULLIF(dw.total_deploys, 0), 0)) AS cfr_pct FROM _adoption_weekly AS aw LEFT JOIN _deployments_weekly AS dw ON aw.week_start = dw.week_start WHERE dw.total_deploys > 0), _tier_avg AS (SELECT tier, AVG(cfr_pct) AS avg_cfr FROM _adoption_cfr GROUP BY tier) SELECT ROUND(CAST(((SELECT avg_cfr FROM _tier_avg WHERE tier = 'high') - (SELECT avg_cfr FROM _tier_avg WHERE tier = 'low')) AS NUMERIC) / NULLIF(NULLIF((SELECT avg_cfr FROM _tier_avg WHERE tier = 'low'), 0), 0) * 100, 1) AS change_pct", + "refId": "A" + } + ], + "title": "CFR Change (High vs Low Adoption)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 400, + "panels": [], + "title": "Recovery Time (MTTR) Impact", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Dual-axis chart showing GitHub Copilot adoption percentage and Mean Time to Recovery (MTTR) over time. MTTR should decrease (improve) as adoption increases if Copilot helps faster incident resolution.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Adoption %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "MTTR (hours)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "h" + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 51, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day') SELECT week_start AS time, ROUND(adoption_pct, 1) AS \"Adoption %\" FROM _adoption_weekly ORDER BY week_start NULLS FIRST", + "refId": "Adoption" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _incidents_weekly AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG((EXTRACT(EPOCH FROM (resolution_date - created_date))/3600)) AS mttr_hours FROM issues WHERE type = 'INCIDENT' AND NOT resolution_date IS NULL AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day') SELECT week_start AS time, ROUND(mttr_hours, 1) AS \"MTTR (hours)\" FROM _incidents_weekly ORDER BY week_start NULLS FIRST", + "refId": "MTTR" + } + ], + "title": "Adoption vs MTTR", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Mean Time to Recovery grouped by GitHub Copilot adoption tier. Lower MTTR at higher adoption indicates Copilot helps faster incident resolution.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "red", + "value": 72 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 64 + }, + "id": 52, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "transformations": [ + { + "id": "rowsToFields", + "options": { + "labelField": "Adoption Tier", + "valueField": "MTTR (hours)" + } + } + ], + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct, CASE WHEN AVG(adoption_pct) < 25 THEN '1: <25%' WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%' WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%' ELSE '4: >75%' END AS adoption_tier FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _incidents_weekly AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG((EXTRACT(EPOCH FROM (resolution_date - created_date))/3600)) AS mttr_hours FROM issues WHERE type = 'INCIDENT' AND NOT resolution_date IS NULL AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day') SELECT aw.adoption_tier AS \"Adoption Tier\", ROUND(CAST(AVG(iw.mttr_hours) AS DECIMAL), 1) AS \"MTTR (hours)\" FROM _adoption_weekly AS aw LEFT JOIN _incidents_weekly AS iw ON aw.week_start = iw.week_start WHERE NOT iw.mttr_hours IS NULL GROUP BY aw.adoption_tier ORDER BY aw.adoption_tier NULLS FIRST", + "refId": "A" + } + ], + "title": "MTTR by Adoption Tier", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient (r) between GitHub Copilot adoption % and MTTR. NEGATIVE values are good - they indicate higher adoption correlates with faster recovery.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -0.3 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 64 + }, + "id": 53, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _incidents_weekly AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG((EXTRACT(EPOCH FROM (resolution_date - created_date))/3600)) AS mttr_hours FROM issues WHERE type = 'INCIDENT' AND NOT resolution_date IS NULL AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT aw.adoption_pct, iw.mttr_hours FROM _adoption_weekly AS aw INNER JOIN _incidents_weekly AS iw ON aw.week_start = iw.week_start) SELECT ROUND(CAST(CAST((COUNT(*) * SUM(adoption_pct * mttr_hours) - SUM(adoption_pct) * SUM(mttr_hours)) AS NUMERIC) / NULLIF(NULLIF(SQRT((COUNT(*) * SUM(adoption_pct * adoption_pct) - POWER(SUM(adoption_pct), 2)) * (COUNT(*) * SUM(mttr_hours * mttr_hours) - POWER(SUM(mttr_hours), 2))), 0), 0) AS DECIMAL), 2) AS correlation_r FROM _joined", + "refId": "A" + } + ], + "title": "Adoption vs MTTR (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage change in MTTR between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate faster recovery.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -20 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 68 + }, + "id": 54, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _incidents_weekly AS (SELECT CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, AVG((EXTRACT(EPOCH FROM (resolution_date - created_date))/3600)) AS mttr_hours FROM issues WHERE type = 'INCIDENT' AND NOT resolution_date IS NULL AND $__timeFilter(resolution_date) GROUP BY CAST(resolution_date AS DATE) - (EXTRACT(ISODOW FROM CAST(resolution_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_mttr AS (SELECT CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier, iw.mttr_hours FROM _adoption_weekly AS aw INNER JOIN _incidents_weekly AS iw ON aw.week_start = iw.week_start), _tier_avg AS (SELECT tier, AVG(mttr_hours) AS avg_mttr FROM _adoption_mttr GROUP BY tier) SELECT ROUND(CAST(((SELECT avg_mttr FROM _tier_avg WHERE tier = 'high') - (SELECT avg_mttr FROM _tier_avg WHERE tier = 'low')) AS NUMERIC) / NULLIF(NULLIF((SELECT avg_mttr FROM _tier_avg WHERE tier = 'low'), 0), 0) * 100, 1) AS change_pct", + "refId": "A" + } + ], + "title": "MTTR Change (High vs Low Adoption)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 500, + "panels": [], + "title": "Code Review Time Impact", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code review time (time from first review request to merge) grouped by GitHub Copilot adoption tier. Shorter review times at higher adoption may indicate Copilot-assisted code is easier to review.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "red", + "value": 48 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 73 + }, + "id": 61, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "transformations": [ + { + "id": "rowsToFields", + "options": { + "labelField": "Adoption Tier", + "valueField": "Review Time (hours)" + } + } + ], + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct, CASE WHEN AVG(adoption_pct) < 25 THEN '1: <25%' WHEN AVG(adoption_pct) < 50 THEN '2: 25-50%' WHEN AVG(adoption_pct) < 75 THEN '3: 50-75%' ELSE '4: >75%' END AS adoption_tier FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_review_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_review_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) AND pr_review_time > 0 GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT aw.adoption_tier AS \"Adoption Tier\", ROUND(CAST(AVG(prw.avg_review_time_hours) AS DECIMAL), 1) AS \"Review Time (hours)\" FROM _adoption_weekly AS aw LEFT JOIN _pr_review_weekly AS prw ON aw.week_start = prw.week_start WHERE NOT prw.avg_review_time_hours IS NULL GROUP BY aw.adoption_tier ORDER BY aw.adoption_tier NULLS FIRST", + "refId": "A" + } + ], + "title": "Review Time by Adoption Tier", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Dual-axis chart showing GitHub Copilot adoption % and average code review time over time. Look for patterns indicating whether Copilot-assisted PRs get reviewed faster.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Adoption %" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Review Time (h)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "h" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 73 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_review_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_review_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) AND pr_review_time > 0 GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day') SELECT EXTRACT(EPOCH FROM aw.week_start) AS time_sec, aw.adoption_pct AS \"Adoption %\", ROUND(prw.avg_review_time_hours, 1) AS \"Review Time (h)\" FROM _adoption_weekly AS aw LEFT JOIN _pr_review_weekly AS prw ON aw.week_start = prw.week_start ORDER BY aw.week_start NULLS FIRST", + "refId": "A" + } + ], + "title": "Adoption vs Review Time Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage change in code review time between low adoption (<50%) and high adoption (\u226550%) periods. Negative values indicate faster reviews.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -10 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 77 + }, + "id": 63, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_review_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_review_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) AND pr_review_time > 0 GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _adoption_review AS (SELECT CASE WHEN aw.adoption_pct < 50 THEN 'low' ELSE 'high' END AS tier, prw.avg_review_time_hours FROM _adoption_weekly AS aw INNER JOIN _pr_review_weekly AS prw ON aw.week_start = prw.week_start), _tier_avg AS (SELECT tier, AVG(avg_review_time_hours) AS avg_review FROM _adoption_review GROUP BY tier) SELECT ROUND(CAST(((SELECT avg_review FROM _tier_avg WHERE tier = 'high') - (SELECT avg_review FROM _tier_avg WHERE tier = 'low')) AS NUMERIC) / NULLIF(NULLIF((SELECT avg_review FROM _tier_avg WHERE tier = 'low'), 0), 0) * 100, 1) AS change_pct", + "refId": "A" + } + ], + "title": "Review Time Change (High vs Low Adoption)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient (r) between GitHub Copilot adoption % and code review time. NEGATIVE values indicate higher adoption correlates with faster reviews.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -0.3 + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 73 + }, + "id": 64, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(day) GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' AND $__timeFilter(date)) AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _pr_review_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_review_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_review_time_hours FROM project_pr_metrics WHERE ('${project:sqlstring}' = '' OR project_name::text = '${project:sqlstring}') AND $__timeFilter(pr_merged_date) AND pr_review_time > 0 GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT aw.adoption_pct, prw.avg_review_time_hours FROM _adoption_weekly AS aw INNER JOIN _pr_review_weekly AS prw ON aw.week_start = prw.week_start) SELECT ROUND(CAST(CAST((COUNT(*) * SUM(adoption_pct * avg_review_time_hours) - SUM(adoption_pct) * SUM(avg_review_time_hours)) AS NUMERIC) / NULLIF(NULLIF(SQRT((COUNT(*) * SUM(adoption_pct * adoption_pct) - POWER(SUM(adoption_pct), 2)) * (COUNT(*) * SUM(avg_review_time_hours * avg_review_time_hours) - POWER(SUM(avg_review_time_hours), 2))), 0), 0) AS DECIMAL), 2) AS correlation_r FROM _joined", + "refId": "A" + } + ], + "title": "Adoption vs Review Time (r)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 81 + }, + "id": 600, + "panels": [], + "title": "Code Quality Impact (Optional - Requires SonarQube)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Bug count per file by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 82 + }, + "id": 80, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _copilot_adoption AS (SELECT metric_date, ROUND(CAST(active_users * 100.0 AS NUMERIC) / NULLIF(NULLIF(total_seats, 0), 0), 1) AS adoption_pct FROM (SELECT day AS metric_date, SUM(daily_active_users) AS active_users, (SELECT COUNT(*) FROM _tool_copilot_seats AS s WHERE ('${connection_id}' = '' OR s.connection_id::text = '${connection_id}') AND COALESCE(s.organization, '') = COALESCE((SELECT sc.organization FROM _tool_copilot_scopes AS sc WHERE ('${connection_id}' = '' OR sc.connection_id::text = '${connection_id}') AND sc.id = '${scope_id}' LIMIT 1), '')) AS total_seats FROM _tool_copilot_enterprise_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}' GROUP BY day UNION ALL SELECT date AS metric_date, total_active_users AS active_users, seat_total AS total_seats FROM _tool_copilot_org_daily_metrics WHERE ('${connection_id}' = '' OR connection_id::text = '${connection_id}') AND scope_id = '${scope_id}') AS _unified), _adoption_weekly AS (SELECT metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day' AS week_start, AVG(adoption_pct) AS adoption_pct FROM _copilot_adoption GROUP BY metric_date - (EXTRACT(ISODOW FROM metric_date) - 1) * INTERVAL '1 day'), _adoption_tiers AS (SELECT week_start, CASE WHEN adoption_pct < 25 THEN '<25%' WHEN adoption_pct < 50 THEN '25-50%' WHEN adoption_pct < 75 THEN '50-75%' ELSE '>75%' END AS tier FROM _adoption_weekly) SELECT 'N/A - Configure SonarQube' AS \"Tier\", 0 AS \"Bugs/File\" FROM (SELECT 1) AS d WHERE NOT EXISTS(SELECT 1 FROM cq_file_metrics LIMIT 1)", + "refId": "A" + } + ], + "title": "Bugs per File by Adoption", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code smell count per file by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 82 + }, + "id": 81, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'N/A - Configure SonarQube' AS \"Tier\", 0 AS \"Code Smells/File\" FROM (SELECT 1) AS d WHERE NOT EXISTS(SELECT 1 FROM cq_file_metrics LIMIT 1)", + "refId": "A" + } + ], + "title": "Code Smells by Adoption", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code complexity by GitHub Copilot adoption tier. Lower is better. Requires SonarQube integration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 25 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 82 + }, + "id": 82, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'N/A - Configure SonarQube' AS \"Tier\", 0 AS \"Complexity\" FROM (SELECT 1) AS d WHERE NOT EXISTS(SELECT 1 FROM cq_file_metrics LIMIT 1)", + "refId": "A" + } + ], + "title": "Complexity by Adoption", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code coverage percentage by GitHub Copilot adoption tier. Higher is better. Requires SonarQube integration.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 82 + }, + "id": 83, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'N/A - Configure SonarQube' AS \"Tier\", 0 AS \"Coverage\" FROM (SELECT 1) AS d WHERE NOT EXISTS(SELECT 1 FROM cq_file_metrics LIMIT 1)", + "refId": "A" + } + ], + "title": "Coverage by Adoption", + "type": "bargauge" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "copilot", + "devlake", + "impact", + "dora" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY 1 NULLS FIRST", + "hide": 0, + "includeAll": false, + "label": "Connection ID", + "multi": false, + "name": "connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC NULLS LAST", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${connection_id}' AS BIGINT) ORDER BY 1 NULLS FIRST", + "hide": 0, + "includeAll": false, + "label": "Scope ID (Organization)", + "multi": false, + "name": "scope_id", + "options": [], + "query": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${connection_id}' AS BIGINT) ORDER BY 1 NULLS FIRST", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT project_name FROM project_pr_metrics ORDER BY 1 NULLS FIRST", + "hide": 0, + "includeAll": false, + "label": "Project", + "multi": false, + "name": "project", + "options": [], + "query": "SELECT DISTINCT project_name FROM project_pr_metrics ORDER BY 1 NULLS FIRST", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "2023", + "value": "2023" + }, + "hide": 0, + "includeAll": false, + "label": "DORA Report", + "multi": false, + "name": "dora_report", + "options": [ + { + "selected": false, + "text": "2021", + "value": "2021" + }, + { + "selected": true, + "text": "2023", + "value": "2023" + } + ], + "query": "2021, 2023", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "GitHub Copilot + DORA Correlation", + "uid": "copilot_impact-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/_archive/EeGitHubReleaseQualityAndContributionAnalysis.json b/grafana/dashboards/postgresql/github-release-quality-and-contribution-analysis.json similarity index 56% rename from grafana/_archive/EeGitHubReleaseQualityAndContributionAnalysis.json rename to grafana/dashboards/postgresql/github-release-quality-and-contribution-analysis.json index 02bb00ec3c5..39765d18d17 100644 --- a/grafana/_archive/EeGitHubReleaseQualityAndContributionAnalysis.json +++ b/grafana/dashboards/postgresql/github-release-quality-and-contribution-analysis.json @@ -15,21 +15,33 @@ "editable": true, "gnetId": null, "graphTooltip": 0, - "id": 21, - "iteration": 1668492584684, + "id": 22, + "iteration": 1682062991983, "links": [], "panels": [ { "datasource": null, "gridPos": { - "h": 1, - "w": 24, + "h": 3, + "w": 13, "x": 0, "y": 0 }, - "id": 49, - "title": "Row title", - "type": "row" + "id": 57, + "links": [], + "options": { + "content": "- Use Cases: This dashboard can be used to track bugs.\n- Data Source Required: GitHub ([transformation](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional) required. Additional settings are required to get version data).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" }, { "collapsed": false, @@ -38,7 +50,7 @@ "h": 1, "w": 24, "x": 0, - "y": 1 + "y": 3 }, "id": 45, "panels": [], @@ -46,7 +58,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -69,9 +81,10 @@ "h": 7, "w": 7, "x": 0, - "y": 2 + "y": 4 }, "id": 15, + "links": [], "options": { "displayLabels": [ "name", @@ -101,10 +114,11 @@ { "format": "table", "group": [], + "hide": false, "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the bug distribution in major versions\nwith bugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/tags/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.type, i.title, i.description\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\tand i.type = 'BUG'\n)\n\n\nSELECT \n\tconcat(SUBSTRING_INDEX(biet.tag_name,'.',2), '.x') as minor_version,\n\tcount(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nGROUP BY 1", + "rawSql": "/* Get the bug distribution in major versions */ WITH bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/tags/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.type, i.title, i.description FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND i.type = 'BUG') SELECT SPLIT_PART(biet.tag_name, '.', 3) || '.x' AS minor_version, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet GROUP BY 1", "refId": "A", "select": [ [ @@ -132,7 +146,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -203,9 +217,10 @@ "h": 7, "w": 17, "x": 7, - "y": 2 + "y": 4 }, "id": 29, + "links": [], "options": { "barWidth": 0.7, "groupWidth": 0.3, @@ -230,7 +245,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the number of fixed bugs in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n -- distinct new_ref_id, old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_bugs_of_tags as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'tags/', -1) as tag_name, \n\t\t-- SUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\tcount(*) as bug_count\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n\t-- GROUP BY 1,2\n\t GROUP BY 1\n),\n\n_combine_pr as (\n select pull_request_id as id, commit_sha from pull_request_commits left join pull_requests p on pull_request_commits.pull_request_id = p.id\n where p.base_repo_id in ($repo_id)\n union\n select id, merge_commit_sha as commit_sha from pull_requests where base_repo_id in ($repo_id)\n),\n\n_commit_count_of_pr as(\n select\n SUBSTRING_INDEX(rcd.new_ref_id,'tags/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) as repo_id,\n pr.id as pull_request_id,\n count(c.sha) as commit_count\n FROM \n refs_commits_diffs rcd\n\t\tleft join commits c on rcd.commit_sha = c.sha\n\t\t-- left join pull_request_commits prc on c.sha = prc.commit_sha\n\t\tleft join _combine_pr pr on c.sha = pr.commit_sha\n\twhere\n\t\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT SUBSTRING_INDEX(new_ref_id,':', -1) FROM _last_5_tags)\n\tgroup by 1,2,3\n),\n\n_pr_worktype as(\n select\n distinct pri.pull_request_id,i.type\n from\n pull_request_issues pri\n left join pull_requests pr on pri.pull_request_id = pr.id\n left join issues i on pri.issue_id = i.id\n where \n i.issue_key != 0\n),\n\n_pr_elco_and_worktype as(\n select\n ccop.tag_name, \n sum(case when pw.type = 'BUG' then commit_count else 0 end)/sum(commit_count) as cost_percentage\n from \n _commit_count_of_pr ccop\n left join _pr_worktype pw on ccop.pull_request_id = pw.pull_request_id\n GROUP BY 1\n)\n\nSELECT \n\tbot.tag_name,\n\tbot.bug_count,\n\tpeaw.cost_percentage as \"cost_percentage(bugfixing commits/total commits)\"\nFROM \n\t_bugs_of_tags bot\n\tjoin _pr_elco_and_worktype peaw on bot.tag_name = peaw.tag_name\nORDER BY 1", + "rawSql": "/* Get the number of fixed bugs in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id /* distinct new_ref_id, old_ref_id */ FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _bugs_of_tags AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'tags/', 1)) AS tag_name, COUNT(*) AS bug_count /* SPLIT_PART(rid.new_ref_id, ':', 3) as repo_id, */ FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG' /* GROUP BY 1, 2 */ GROUP BY 1), _combine_pr AS (SELECT pull_request_id AS id, commit_sha FROM pull_request_commits LEFT JOIN pull_requests AS p ON pull_request_commits.pull_request_id = p.id WHERE ('${repo_id:csv}' = '' OR p.base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) UNION SELECT id, merge_commit_sha AS commit_sha FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))), _commit_count_of_pr AS (SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'tags/', 1)) AS tag_name, SPLIT_PART(rcd.new_ref_id, ':', 4) AS repo_id, pr.id AS pull_request_id, COUNT(c.sha) AS commit_count FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha /* left join pull_request_commits prc on c.sha = prc.commit_sha */ LEFT JOIN _combine_pr AS pr ON c.sha = pr.commit_sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) FROM _last_5_tags) GROUP BY 1, 2, 3), _pr_worktype AS (SELECT DISTINCT pri.pull_request_id, i.type FROM pull_request_issues AS pri LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id LEFT JOIN issues AS i ON pri.issue_id = i.id WHERE i.issue_key <> '0'), _pr_elco_and_worktype AS (SELECT ccop.tag_name, CAST(SUM(CASE WHEN pw.type = 'BUG' THEN commit_count ELSE 0 END) AS NUMERIC) / NULLIF(SUM(commit_count), 0) AS cost_percentage FROM _commit_count_of_pr AS ccop LEFT JOIN _pr_worktype AS pw ON ccop.pull_request_id = pw.pull_request_id GROUP BY 1) SELECT bot.tag_name, bot.bug_count, peaw.cost_percentage AS \"cost_percentage(bugfixing commits/total commits)\" FROM _bugs_of_tags AS bot JOIN _pr_elco_and_worktype AS peaw ON bot.tag_name = peaw.tag_name ORDER BY 1 NULLS FIRST", "refId": "A", "select": [ [ @@ -258,7 +273,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -274,85 +289,16 @@ }, "mappings": [] }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "dev_eq" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts UNKNOWN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts REQUIREMENT" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts BUG" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, - "y": 9 + "y": 11 }, "id": 55, + "links": [], "options": { "displayLabels": [ "percent" @@ -385,7 +331,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the severity distribution in bugs\n-- Get the work-type distribution in the last n tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_n_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 1\n),\n\nbugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/tags/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.type, i.title, i.description, i.severity \n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand i.type = 'BUG'\n)\n\nSELECT \n concat(biet.tag_name, \" \", case when biet.severity != '' then biet.severity else 'UNKNOWN' end) as severity,\n count(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nGROUP BY 1\n\n-- SELECT \n-- \t case when biet.severity != '' then biet.severity else 'UNKNOWN' end as severity,\n-- \tcount(*) as bug_count\n-- FROM \n-- \tbugs_in_each_tag biet\n-- GROUP BY biet.severity", + "rawSql": "/* Get the severity distribution in bugs */ /* Get the work-type distribution in the last n tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_n_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1), bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/tags/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.type, i.title, i.description, i.severity FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_n_tags) AND i.type = 'BUG') SELECT biet.tag_name || ' ' || CASE WHEN biet.severity <> '' THEN biet.severity ELSE 'UNKNOWN' END AS severity, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet GROUP BY 1", "refId": "A", "select": [ [ @@ -413,7 +359,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -505,9 +451,10 @@ "h": 7, "w": 8, "x": 8, - "y": 9 + "y": 11 }, "id": 53, + "links": [], "options": { "displayLabels": [ "percent" @@ -540,7 +487,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the severity distribution in bugs\n-- Get the work-type distribution in the last n tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_n_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 1,1\n),\n\nbugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/tags/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.type, i.title, i.description, i.severity \n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand i.type = 'BUG'\n)\n\nSELECT \n concat(biet.tag_name, \" \", case when biet.severity != '' then biet.severity else 'UNKNOWN' end) as severity,\n count(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nGROUP BY 1\n\n-- SELECT \n-- \t case when biet.severity != '' then biet.severity else 'UNKNOWN' end as severity,\n-- \tcount(*) as bug_count\n-- FROM \n-- \tbugs_in_each_tag biet\n-- GROUP BY biet.severity", + "rawSql": "/* Get the severity distribution in bugs */ /* Get the work-type distribution in the last n tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_n_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1 OFFSET 1), bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/tags/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.type, i.title, i.description, i.severity FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_n_tags) AND i.type = 'BUG') SELECT biet.tag_name || ' ' || CASE WHEN biet.severity <> '' THEN biet.severity ELSE 'UNKNOWN' END AS severity, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet GROUP BY 1", "refId": "A", "select": [ [ @@ -568,7 +515,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -584,85 +531,16 @@ }, "mappings": [] }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "dev_eq" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts UNKNOWN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts REQUIREMENT" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts BUG" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "green", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, - "y": 9 + "y": 11 }, "id": 51, + "links": [], "options": { "displayLabels": [ "percent" @@ -695,7 +573,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the severity distribution in bugs\n-- Get the work-type distribution in the last n tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_n_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 2,1\n),\n\nbugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/tags/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.type, i.title, i.description, i.severity \n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_n_tags)\n\t\tand i.type = 'BUG'\n)\n\nSELECT \n concat(biet.tag_name, \" \", case when biet.severity != '' then biet.severity else 'UNKNOWN' end) as severity,\n count(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nGROUP BY 1\n\n-- SELECT \n-- \t case when biet.severity != '' then biet.severity else 'UNKNOWN' end as severity,\n-- \tcount(*) as bug_count\n-- FROM \n-- \tbugs_in_each_tag biet\n-- GROUP BY biet.severity", + "rawSql": "/* Get the severity distribution in bugs */ /* Get the work-type distribution in the last n tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_n_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1 OFFSET 2), bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/tags/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.type, i.title, i.description, i.severity FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_n_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_n_tags) AND i.type = 'BUG') SELECT biet.tag_name || ' ' || CASE WHEN biet.severity <> '' THEN biet.severity ELSE 'UNKNOWN' END AS severity, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet GROUP BY 1", "refId": "A", "select": [ [ @@ -723,7 +601,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -841,14 +719,15 @@ "h": 7, "w": 24, "x": 0, - "y": 16 + "y": 18 }, "id": 43, + "links": [], "options": { "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -856,7 +735,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the number of fixed bugs in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n)\n\t\nselect distinct\n\tb.name as repo_name,\n\tSUBSTRING_INDEX(rid.new_ref_id,'tags/', -1) as tag_name, \n\ti.issue_key as issue_key,\n\ti.title,\n\ti.assignee_name,\n\ti.lead_time_minutes/1440 as lead_time_in_days,\n\tconcat(b.url,'/',i.issue_key) as url\nfrom\n\trefs_issues_diffs rid\n\tleft join issues i on rid.issue_id = i.id\n\tjoin boards b on SUBSTRING_INDEX(rid.new_ref_id,':', 3) = b.id\nwhere\n\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT SUBSTRING_INDEX(new_ref_id,':', -1) FROM _last_5_tags)\n\tand i.type = 'BUG'\norder by tag_name desc", + "rawSql": "/* Get the number of fixed bugs in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5) SELECT DISTINCT b.name AS repo_name, REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'tags/', 1)) AS tag_name, i.issue_key AS issue_key, i.title, i.assignee_name, CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS lead_time_in_days, b.url || '/' || i.issue_key AS url FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id JOIN boards AS b ON SPLIT_PART(rid.new_ref_id, ':', 4) = b.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) FROM _last_5_tags) AND i.type = 'BUG' ORDER BY tag_name DESC NULLS LAST", "refId": "A", "select": [ [ @@ -884,7 +763,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -907,9 +786,10 @@ "h": 7, "w": 11, "x": 0, - "y": 23 + "y": 25 }, "id": 30, + "links": [], "options": { "displayLabels": [ "percent", @@ -942,7 +822,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Component distribution of bugs fixed in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\nbugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.component, i.severity, i.title, i.description\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n)\n\n\nSELECT\n\tcase when component = '' then 'unlabeled' else 'labeled' end as component,\n\tcount(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nGROUP BY 1", + "rawSql": "/* Component distribution of bugs fixed in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.component, i.severity, i.title, i.description FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG') SELECT CASE WHEN component = '' THEN 'unlabeled' ELSE 'labeled' END AS component, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet GROUP BY 1", "refId": "A", "select": [ [ @@ -970,7 +850,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -993,9 +873,10 @@ "h": 7, "w": 13, "x": 11, - "y": 23 + "y": 25 }, "id": 31, + "links": [], "options": { "displayLabels": [ "percent", @@ -1028,7 +909,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Component distribution of bugs fixed in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\nbugs_in_each_tag as(\n\tselect \n\t\tSUBSTRING_INDEX(rid.new_ref_id,'refs/', -1) as tag_name, \n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) as repo_id,\n\t\ti.issue_key, i.component, i.severity, i.title, i.description\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n)\n\n\nSELECT\n\tcomponent,\n\tcount(*) as bug_count\nFROM \n\tbugs_in_each_tag biet\nwhere \n component != ''\nGROUP BY 1", + "rawSql": "/* Component distribution of bugs fixed in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), bugs_in_each_tag AS (SELECT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'refs/', 1)) AS tag_name, SPLIT_PART(rid.new_ref_id, ':', 4) AS repo_id, i.issue_key, i.component, i.severity, i.title, i.description FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG') SELECT component, COUNT(*) AS bug_count FROM bugs_in_each_tag AS biet WHERE component <> '' GROUP BY 1", "refId": "A", "select": [ [ @@ -1056,7 +937,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1089,9 +970,10 @@ "h": 7, "w": 6, "x": 0, - "y": 30 + "y": 32 }, "id": 23, + "links": [], "options": { "colorMode": "value", "graphMode": "area", @@ -1107,7 +989,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1115,7 +997,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the % of contributors who fixed 80% of bugs in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_bugs as(\n\tselect \n\t\ti.issue_key, i.type, i.severity, i.title, i.description,\n\t\tpr.id, pr.author_name as pr_author, pr.created_date,\n\t\trank() over(partition by i.id order by pr.created_date asc) as pr_rank\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\t\tleft join pull_request_issues pri on i.id = pri.issue_id\n\t\tleft join pull_requests pr on pri.pull_request_id = pr.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n\torder by i.issue_key\n),\n\n_bug_fixed_count as(\n SELECT \n pr_author,\n count(*) bug_fixed_count\n FROM _bugs\n WHERE pr_rank = 1\n GROUP BY 1\n),\n\n_bug_fixed_count_running_total as(\n SELECT \n *, \n sum(bug_fixed_count) OVER (Order by bug_fixed_count desc) AS running_total\n FROM \n _bug_fixed_count\n),\n\n_percentile as(\n SELECT \n pr_author,\n bug_fixed_count,\n running_total/sum(bug_fixed_count) OVER () AS cumulative_percentage\n FROM \n _bug_fixed_count_running_total\n)\n\n\nSELECT \n count(case when cumulative_percentage <= 0.8 then pr_author else null end)/count(*) as \"% of contributors who fixed 80% of the bugs\"\nFROM _percentile", + "rawSql": "/* Get the % of contributors who fixed 80% of bugs in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _bugs AS (SELECT i.issue_key, i.type, i.severity, i.title, i.description, pr.id, pr.author_name AS pr_author, pr.created_date, RANK() OVER (PARTITION BY i.id ORDER BY pr.created_date ASC NULLS FIRST) AS pr_rank FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id LEFT JOIN pull_request_issues AS pri ON i.id = pri.issue_id LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG' ORDER BY i.issue_key NULLS FIRST), _bug_fixed_count AS (SELECT pr_author, COUNT(*) AS bug_fixed_count FROM _bugs WHERE pr_rank = 1 GROUP BY 1), _bug_fixed_count_running_total AS (SELECT *, SUM(bug_fixed_count) OVER (ORDER BY bug_fixed_count DESC NULLS LAST) AS running_total FROM _bug_fixed_count), _percentile AS (SELECT pr_author, bug_fixed_count, CAST(running_total AS NUMERIC) / NULLIF(SUM(bug_fixed_count) OVER (), 0) AS cumulative_percentage FROM _bug_fixed_count_running_total) SELECT CAST(COUNT(CASE WHEN cumulative_percentage <= 0.8 THEN pr_author ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"% of contributors who fixed 80% of the bugs\" FROM _percentile", "refId": "A", "select": [ [ @@ -1143,7 +1025,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1185,9 +1067,10 @@ "h": 7, "w": 18, "x": 6, - "y": 30 + "y": 32 }, "id": 18, + "links": [], "options": { "barWidth": 0.5, "groupWidth": 0.7, @@ -1212,7 +1095,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the bug fixer distribution in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_bugs as(\n\tselect \n\t\ti.issue_key, i.type, i.severity, i.title, i.description,\n\t\tpr.id, pr.author_name as pr_author, pr.created_date,\n\t\trank() over(partition by i.id order by pr.created_date asc) as pr_rank\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\t\tleft join pull_request_issues pri on i.id = pri.issue_id\n\t\tleft join pull_requests pr on pri.pull_request_id = pr.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n\torder by i.issue_key\n)\n\nSELECT \n pr_author,\n count(*) bug_fixed_count\nFROM _bugs\nWHERE pr_rank = 1\nGROUP BY 1\nORDER BY 2 desc\nlimit 10", + "rawSql": "/* Get the bug fixer distribution in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _bugs AS (SELECT i.issue_key, i.type, i.severity, i.title, i.description, pr.id, pr.author_name AS pr_author, pr.created_date, RANK() OVER (PARTITION BY i.id ORDER BY pr.created_date ASC NULLS FIRST) AS pr_rank FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id LEFT JOIN pull_request_issues AS pri ON i.id = pri.issue_id LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG' ORDER BY i.issue_key NULLS FIRST) SELECT pr_author, COUNT(*) AS bug_fixed_count FROM _bugs WHERE pr_rank = 1 GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 10", "refId": "A", "select": [ [ @@ -1240,7 +1123,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1269,9 +1152,16 @@ "h": 6, "w": 4, "x": 0, - "y": 37 + "y": 39 }, "id": 33, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "colorMode": "value", "graphMode": "area", @@ -1287,7 +1177,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1295,7 +1185,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the avg bug age in history\nselect \n avg(lead_time_minutes)/1440 as average_bug_age\nfrom issues\nleft join board_issues bi on issues.id = bi.issue_id\nwhere \n type = 'BUG'\n and status = 'DONE'\n and bi.board_id in ($repo_id)", + "rawSql": "/* Get the avg bug age in history */ SELECT CAST(AVG(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS average_bug_age FROM issues LEFT JOIN board_issues AS bi ON issues.id = bi.issue_id WHERE type = 'BUG' AND status = 'DONE' AND ('${repo_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", "refId": "A", "select": [ [ @@ -1323,7 +1213,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1365,9 +1255,16 @@ "h": 6, "w": 20, "x": 4, - "y": 37 + "y": 39 }, "id": 32, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "barWidth": 0.7, "groupWidth": 0.3, @@ -1392,7 +1289,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the bug age in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_bugs as(\n\tselect distinct\n\t\tSUBSTRING_INDEX(rid.new_ref_id,'tags/', -1) as tag_name,\n\t\ti.id,\n\t\ti.lead_time_minutes\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\t\tleft join pull_request_issues pri on i.id = pri.issue_id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n),\n\n_bugs_percentile as(\n select \n *,\n percent_rank() over (partition by tag_name order by lead_time_minutes) as percentile\n from _bugs order by 1\n),\n\n_avg_bug_age as(\n select \n tag_name,\n avg(lead_time_minutes)/1440 as average_bug_age\n from _bugs_percentile\n group by 1\n),\n\n_50th_bug_age as(\n select \n tag_name,\n min(lead_time_minutes)/1440 as \"50th_bug_age\"\n from _bugs_percentile\n where percentile >= 0.5\n group by 1\n)\n\nselect \n aba.*,\n eba.50th_bug_age\nfrom \n _avg_bug_age aba\n join _50th_bug_age eba on aba.tag_name = eba.tag_name", + "rawSql": "/* Get the bug age in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _bugs AS (SELECT DISTINCT REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'tags/', 1)) AS tag_name, i.id, i.lead_time_minutes FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id LEFT JOIN pull_request_issues AS pri ON i.id = pri.issue_id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG'), _bugs_percentile AS (SELECT *, PERCENT_RANK() OVER (PARTITION BY tag_name ORDER BY lead_time_minutes NULLS FIRST) AS percentile FROM _bugs ORDER BY 1 NULLS FIRST), _avg_bug_age AS (SELECT tag_name, CAST(AVG(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS average_bug_age FROM _bugs_percentile GROUP BY 1), _50th_bug_age AS (SELECT tag_name, CAST(MIN(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS \"50th_bug_age\" FROM _bugs_percentile WHERE percentile >= 0.5 GROUP BY 1) SELECT aba.*, eba.\"50th_bug_age\" FROM _avg_bug_age AS aba JOIN _50th_bug_age AS eba ON aba.tag_name = eba.tag_name", "refId": "A", "select": [ [ @@ -1420,7 +1317,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1449,9 +1346,16 @@ "h": 6, "w": 4, "x": 0, - "y": 43 + "y": 45 }, "id": 34, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "colorMode": "value", "graphMode": "area", @@ -1467,7 +1371,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1475,7 +1379,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the 80th perccentile bug age in history\nwith _bugs_percentile as(\n select \n id,\n lead_time_minutes,\n percent_rank() over (order by lead_time_minutes) as percentile\n from issues\n where \n type = 'BUG'\n and status = 'DONE'\n)\n\nselect \n min(lead_time_minutes)/1440 as \"80th_bug_age\"\nfrom \n _bugs_percentile\nwhere \n percentile >= 0.5\n", + "rawSql": "/* Get the 80th perccentile bug age in history */ WITH _bugs_percentile AS (SELECT id, lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes NULLS FIRST) AS percentile FROM issues WHERE type = 'BUG' AND status = 'DONE') SELECT CAST(MIN(lead_time_minutes) AS NUMERIC) / NULLIF(1440, 0) AS \"80th_bug_age\" FROM _bugs_percentile WHERE percentile >= 0.5", "refId": "A", "select": [ [ @@ -1503,7 +1407,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1621,14 +1525,21 @@ "h": 6, "w": 20, "x": 4, - "y": 43 + "y": 45 }, "id": 38, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1636,7 +1547,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the bug fixer distribution in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_bugs as(\n\tselect distinct\n\t b.name,\n\t\tSUBSTRING_INDEX(rid.new_ref_id,'tags/', -1) as tag_name,\n\t\ti.issue_key as issue_key,\n i.title,\n i.lead_time_minutes/1440 as lead_time_in_days,\n concat(b.url,'/',i.issue_key) as url\n\tfrom\n\t\trefs_issues_diffs rid\n\t\tleft join issues i on rid.issue_id = i.id\n\t\tleft join pull_request_issues pri on i.id = pri.issue_id\n\t\tjoin boards b on SUBSTRING_INDEX(rid.new_ref_id,':', 3) = b.id\n\twhere\n\t\tSUBSTRING_INDEX(rid.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rid.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand i.type = 'BUG'\n),\n\n_bug_age_rank as(\n select \n *,\n row_number() over (partition by tag_name order by lead_time_in_days desc) as bug_age_rank\n from _bugs\n)\n\nselect \n name,\n tag_name,\n issue_key,\n title,\n lead_time_in_days,\n url\nfrom _bug_age_rank\nwhere bug_age_rank <=10 \norder by tag_name, lead_time_in_days desc", + "rawSql": "/* Get the bug fixer distribution in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _bugs AS (SELECT DISTINCT b.name, REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), 'tags/', 1)) AS tag_name, i.issue_key AS issue_key, i.title, CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS lead_time_in_days, b.url || '/' || i.issue_key AS url FROM refs_issues_diffs AS rid LEFT JOIN issues AS i ON rid.issue_id = i.id LEFT JOIN pull_request_issues AS pri ON i.id = pri.issue_id JOIN boards AS b ON SPLIT_PART(rid.new_ref_id, ':', 4) = b.id WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rid.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rid.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rid.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) AND i.type = 'BUG'), _bug_age_rank AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY tag_name ORDER BY lead_time_in_days DESC NULLS LAST) AS bug_age_rank FROM _bugs) SELECT name, tag_name, issue_key, title, lead_time_in_days, url FROM _bug_age_rank WHERE bug_age_rank <= 10 ORDER BY tag_name NULLS FIRST, lead_time_in_days DESC NULLS LAST", "refId": "A", "select": [ [ @@ -1664,7 +1575,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1742,9 +1653,16 @@ "h": 6, "w": 6, "x": 0, - "y": 49 + "y": 51 }, "id": 35, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "displayLabels": [ "name", @@ -1770,7 +1688,7 @@ "mode": "single" } }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1778,7 +1696,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the avg bug age in history\nwith _avg_bug_age as(\n select \n type,\n avg(lead_time_minutes) as average_bug_age\n from \n issues\n left join board_issues bi on issues.id = bi.issue_id\n where \n type = 'BUG'\n and status = 'DONE'\n and bi.board_id in ($repo_id)\n group by 1\n),\n\n\n_bug_queue_time as(\n select \n i.id,\n abg.average_bug_age,\n TIMESTAMPDIFF(MINUTE,created_date,NOW()) as queue_time,\n case when TIMESTAMPDIFF(MINUTE,created_date,NOW()) >= average_bug_age then \">=avg_bug_age\" else \"= average_bug_age THEN '>=avg_bug_age' ELSE ' 'DONE') SELECT distribution, COUNT(*) AS bug_count FROM _bug_queue_time GROUP BY 1", "refId": "A", "select": [ [ @@ -1806,7 +1724,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -1918,14 +1836,21 @@ "h": 6, "w": 18, "x": 6, - "y": 49 + "y": 51 }, "id": 39, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], "options": { "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -1933,7 +1858,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the queue time of all backlog bugs\nselect \n b.name as repo_name,\n i.issue_key as issue_key,\n i.title,\n i.created_date,\n (TIMESTAMPDIFF(MINUTE, i.created_date,NOW()))/3600 as queue_time_in_days,\n concat(b.url,'/',i.issue_key) as url\nfrom \n issues i\n left join board_issues bi on i.id = bi.issue_id\n left join boards b on bi.board_id = b.id\nwhere\n i.type = 'BUG'\n and i.status != 'DONE'\n and b.id in ($repo_id)\norder by queue_time_in_days desc", + "rawSql": "/* Get the queue time of all backlog bugs */ SELECT b.name AS repo_name, i.issue_key AS issue_key, i.title, i.created_date, CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(3600, 0) AS queue_time_in_days, b.url || '/' || i.issue_key AS url FROM issues AS i LEFT JOIN board_issues AS bi ON i.id = bi.issue_id LEFT JOIN boards AS b ON bi.board_id = b.id WHERE i.type = 'BUG' AND i.status <> 'DONE' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY queue_time_in_days DESC NULLS LAST", "refId": "A", "select": [ [ @@ -1967,7 +1892,7 @@ "h": 1, "w": 24, "x": 0, - "y": 55 + "y": 57 }, "id": 47, "panels": [], @@ -1975,7 +1900,7 @@ "type": "row" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "fieldConfig": { "defaults": { "color": { @@ -2015,9 +1940,10 @@ "h": 7, "w": 6, "x": 0, - "y": 56 + "y": 58 }, "id": 41, + "links": [], "options": { "barWidth": 0.3, "groupWidth": 0.7, @@ -2041,7 +1967,7 @@ "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "-- Get the bug distribution in last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 10\n)\n\nselect \n\tSUBSTRING_INDEX(rcd.new_ref_id,'refs/tags/', -1) as tag_name,\n\tSUBSTRING_INDEX(rcd.old_ref_id,'refs/tags/', -1) as old_tag_name,\n\tcount(*) as commit_count\nfrom\n\trefs_commits_diffs rcd\n\tleft join commits c on rcd.commit_sha = c.sha\nwhere\n\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t-- and rcd.new_ref_id in (select new_ref_id from _last_5_tags)\n\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\ngroup by 1,2\norder by 1", + "rawSql": "/* Get the bug distribution in last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 10) SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'refs/tags/', 1)) AS tag_name, REVERSE(SPLIT_PART(REVERSE(rcd.old_ref_id), 'refs/tags/', 1)) AS old_tag_name, COUNT(*) AS commit_count FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (select new_ref_id from _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1, 2 ORDER BY 1 NULLS FIRST", "refId": "A", "select": [ [ @@ -2069,7 +1995,7 @@ "type": "barchart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "fieldConfig": { "defaults": { "color": { @@ -2137,21 +2063,22 @@ "h": 7, "w": 18, "x": 6, - "y": 56 + "y": 58 }, "id": 42, + "links": [], "options": { "showHeader": true, "sortBy": [] }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", "group": [], "metricColumn": "none", "rawQuery": true, - "rawSql": "-- Get the bug distribution in last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 10\n)\n\nselect \n\tSUBSTRING_INDEX(rcd.new_ref_id,'refs/tags/', -1) as new_tag_name,\n\tSUBSTRING_INDEX(rcd.old_ref_id,'refs/tags/', -1) as compared_tag_name,\n\tc.sha,\n\tc.message,\n\tc.additions,\n\tc.deletions,\n\tc.author_name\nfrom\n\trefs_commits_diffs rcd\n\tleft join commits c on rcd.commit_sha = c.sha\nwhere\n\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t-- and rcd.new_ref_id in (select new_ref_id from _last_5_tags)\n\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\norder by 1 desc", + "rawSql": "/* Get the bug distribution in last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 10) SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'refs/tags/', 1)) AS new_tag_name, REVERSE(SPLIT_PART(REVERSE(rcd.old_ref_id), 'refs/tags/', 1)) AS compared_tag_name, c.sha, c.message, c.additions, c.deletions, c.author_name FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (select new_ref_id from _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) ORDER BY 1 DESC NULLS LAST", "refId": "A", "select": [ [ @@ -2179,7 +2106,7 @@ "type": "table" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2196,55 +2123,16 @@ "mappings": [] }, "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "dev_eq" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": false - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "v22.3.2.2-lts UNKNOWN" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "blue", - "mode": "fixed" - } - } - ] - }, { "matcher": { "id": "byName", - "options": "v22.3.2.2-lts REQUIREMENT" + "options": "v22.3.2.2-lts BUG" }, "properties": [ { "id": "color", "value": { - "fixedColor": "yellow", + "fixedColor": "red", "mode": "fixed" } } @@ -2253,13 +2141,13 @@ { "matcher": { "id": "byName", - "options": "v22.3.2.2-lts BUG" + "options": "v22.3.2.2-lts UNKNOWN" }, "properties": [ { "id": "color", "value": { - "fixedColor": "green", + "fixedColor": "text", "mode": "fixed" } } @@ -2271,9 +2159,10 @@ "h": 7, "w": 8, "x": 0, - "y": 63 + "y": 65 }, "id": 26, + "links": [], "options": { "displayLabels": [ "percent" @@ -2306,7 +2195,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the work-type distribution in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 1\n),\n\n_combine_pr as (\n select pull_request_id as id, commit_sha, p.pull_request_key as pull_request_key from pull_request_commits left join pull_requests p on pull_request_commits.pull_request_id = p.id\n where base_repo_id in ($repo_id)\n union\n select id, merge_commit_sha, pull_request_key as commit_sha from pull_requests where base_repo_id in ($repo_id)\n),\n\n_commit_count_of_pr as(\n select\n SUBSTRING_INDEX(rcd.new_ref_id,'tags/', -1) as tag_name,\n pr.id as pull_request_id,\n -- count(c.sha) as pr_commit_count\n sum(c.dev_eq) as pr_eloc\n FROM \n refs_commits_diffs rcd\n\t\tleft join commits c on rcd.commit_sha = c.sha\n\t\t-- left join pull_request_commits prc on c.sha = prc.commit_sha\n\t\tleft join _combine_pr pr on c.sha = pr.commit_sha\n\twhere\n\t\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\tgroup by 1,2\n),\n\n_pr_issues as(\n select\n pri.pull_request_id,\n pri.issue_id,\n i.issue_key,\n i.type,\n row_number() over(partition by issue_id ORDER by pr.created_date asc) as pr_rank\n from\n pull_request_issues pri\n left join pull_requests pr on pri.pull_request_id = pr.id\n left join issues i on pri.issue_id = i.id\n where \n i.issue_key != 0\n),\n\n_final_results as(\n select\n distinct ccop.*, pi.type\n from \n _commit_count_of_pr ccop\n left join _pr_issues pi on ccop.pull_request_id = pi.pull_request_id\n where pi.pr_rank = 1\n order by 1\n)\n\nSELECT\n tag_name,\n case when type != '' then type else 'UNKNOWN' end as type,\n -- sum(pr_commit_count) as commit_count\n sum(pr_eloc) as dev_eq\nfrom _final_results\ngroup by 1,2", + "rawSql": "/* Get the work-type distribution in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1), _combine_pr AS (SELECT pull_request_id AS id, commit_sha, p.pull_request_key AS pull_request_key FROM pull_request_commits LEFT JOIN pull_requests AS p ON pull_request_commits.pull_request_id = p.id WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) UNION SELECT id, merge_commit_sha, pull_request_key AS commit_sha FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))), _commit_count_of_pr AS (SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'tags/', 1)) AS tag_name, pr.id AS pull_request_id, COUNT(c.sha) AS pr_commit_count FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha /* left join pull_request_commits prc on c.sha = prc.commit_sha */ LEFT JOIN _combine_pr AS pr ON c.sha = pr.commit_sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1, 2), _pr_issues AS (SELECT pri.pull_request_id, pri.issue_id, i.issue_key, i.type, ROW_NUMBER() OVER (PARTITION BY issue_id ORDER BY pr.created_date ASC NULLS FIRST) AS pr_rank FROM pull_request_issues AS pri LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id LEFT JOIN issues AS i ON pri.issue_id = i.id WHERE i.issue_key <> '0'), _final_results AS (SELECT DISTINCT ccop.*, pi.type FROM _commit_count_of_pr AS ccop LEFT JOIN _pr_issues AS pi ON ccop.pull_request_id = pi.pull_request_id WHERE pi.pr_rank = 1 ORDER BY 1 NULLS FIRST) SELECT tag_name, CASE WHEN type <> '' THEN type ELSE 'UNKNOWN' END AS type, SUM(pr_commit_count) AS commit_count FROM _final_results GROUP BY 1, 2", "refId": "A", "select": [ [ @@ -2334,7 +2223,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2350,15 +2239,47 @@ }, "mappings": [] }, - "overrides": [] + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "v22.2.3.5-stable UNKNOWN" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "text", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "v22.2.3.5-stable BUG" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] }, "gridPos": { "h": 7, "w": 8, "x": 8, - "y": 63 + "y": 65 }, "id": 36, + "links": [], "options": { "displayLabels": [ "percent" @@ -2391,7 +2312,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the work-type distribution in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 1,1\n),\n\n\n_combine_pr as (\n select pull_request_id as id, commit_sha, p.pull_request_key as pull_request_key from pull_request_commits left join pull_requests p on pull_request_commits.pull_request_id = p.id\n where base_repo_id in ($repo_id)\n union\n select id, merge_commit_sha, pull_request_key as commit_sha from pull_requests where base_repo_id in ($repo_id)\n),\n\n_commit_count_of_pr as(\n select\n SUBSTRING_INDEX(rcd.new_ref_id,'tags/', -1) as tag_name,\n pr.id as pull_request_id,\n -- count(c.sha) as pr_commit_count\n sum(c.dev_eq) as pr_eloc\n FROM \n refs_commits_diffs rcd\n\t\tleft join commits c on rcd.commit_sha = c.sha\n\t\t-- left join pull_request_commits prc on c.sha = prc.commit_sha\n\t\tleft join _combine_pr pr on c.sha = pr.commit_sha\n\twhere\n\t\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\tgroup by 1,2\n),\n\n_pr_issues as(\n select\n pri.pull_request_id,\n pri.issue_id,\n i.issue_key,\n i.type,\n row_number() over(partition by issue_id ORDER by pr.created_date asc) as pr_rank\n from\n pull_request_issues pri\n left join pull_requests pr on pri.pull_request_id = pr.id\n left join issues i on pri.issue_id = i.id\n where \n i.issue_key != 0\n),\n\n_final_results as(\n select\n distinct ccop.*, pi.type\n from \n _commit_count_of_pr ccop\n left join _pr_issues pi on ccop.pull_request_id = pi.pull_request_id\n where pi.pr_rank = 1\n order by 1\n)\n\nSELECT\n tag_name,\n case when type != '' then type else 'UNKNOWN' end as type,\n -- sum(pr_commit_count) as commit_count\n sum(pr_eloc) as dev_eq\nfrom _final_results\ngroup by 1,2", + "rawSql": "/* Get the work-type distribution in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1 OFFSET 1), _combine_pr AS (SELECT pull_request_id AS id, commit_sha, p.pull_request_key AS pull_request_key FROM pull_request_commits LEFT JOIN pull_requests AS p ON pull_request_commits.pull_request_id = p.id WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) UNION SELECT id, merge_commit_sha, pull_request_key AS commit_sha FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))), _commit_count_of_pr AS (SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'tags/', 1)) AS tag_name, pr.id AS pull_request_id, COUNT(c.sha) AS pr_commit_count FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha /* left join pull_request_commits prc on c.sha = prc.commit_sha */ LEFT JOIN _combine_pr AS pr ON c.sha = pr.commit_sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1, 2), _pr_issues AS (SELECT pri.pull_request_id, pri.issue_id, i.issue_key, i.type, ROW_NUMBER() OVER (PARTITION BY issue_id ORDER BY pr.created_date ASC NULLS FIRST) AS pr_rank FROM pull_request_issues AS pri LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id LEFT JOIN issues AS i ON pri.issue_id = i.id WHERE i.issue_key <> '0'), _final_results AS (SELECT DISTINCT ccop.*, pi.type FROM _commit_count_of_pr AS ccop LEFT JOIN _pr_issues AS pi ON ccop.pull_request_id = pi.pull_request_id WHERE pi.pr_rank = 1 ORDER BY 1 NULLS FIRST) SELECT tag_name, CASE WHEN type <> '' THEN type ELSE 'UNKNOWN' END AS type, SUM(pr_commit_count) AS commit_count FROM _final_results GROUP BY 1, 2", "refId": "A", "select": [ [ @@ -2419,7 +2340,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2439,13 +2360,13 @@ { "matcher": { "id": "byName", - "options": "v22.1.4.30-stable BUG" + "options": "v22.1.4.30-stable UNKNOWN" }, "properties": [ { "id": "color", "value": { - "fixedColor": "dark-yellow", + "fixedColor": "text", "mode": "fixed" } } @@ -2454,13 +2375,13 @@ { "matcher": { "id": "byName", - "options": "v22.1.4.30-stable REQUIREMENT" + "options": "v22.1.4.30-stable BUG" }, "properties": [ { "id": "color", "value": { - "fixedColor": "blue", + "fixedColor": "red", "mode": "fixed" } } @@ -2472,9 +2393,10 @@ "h": 7, "w": 8, "x": 16, - "y": 63 + "y": 65 }, "id": 37, + "links": [], "options": { "displayLabels": [ "percent" @@ -2507,7 +2429,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get the work-type distribution in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 2,1\n),\n\n\n_combine_pr as (\n select pull_request_id as id, commit_sha, p.pull_request_key as pull_request_key from pull_request_commits left join pull_requests p on pull_request_commits.pull_request_id = p.id\n where base_repo_id in ($repo_id)\n union\n select id, merge_commit_sha, pull_request_key as commit_sha from pull_requests where base_repo_id in ($repo_id)\n),\n\n_commit_count_of_pr as(\n select\n SUBSTRING_INDEX(rcd.new_ref_id,'tags/', -1) as tag_name,\n pr.id as pull_request_id,\n -- count(c.sha) as pr_commit_count\n sum(c.dev_eq) as pr_eloc\n FROM \n refs_commits_diffs rcd\n\t\tleft join commits c on rcd.commit_sha = c.sha\n\t\t-- left join pull_request_commits prc on c.sha = prc.commit_sha\n\t\tleft join _combine_pr pr on c.sha = pr.commit_sha\n\twhere\n\t\tSUBSTRING_INDEX(rcd.new_ref_id,':', 3) in ($repo_id)\n\t\t-- and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\t\tand SUBSTRING_INDEX(rcd.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n\tgroup by 1,2\n),\n\n_pr_issues as(\n select\n pri.pull_request_id,\n pri.issue_id,\n i.issue_key,\n i.type,\n row_number() over(partition by issue_id ORDER by pr.created_date asc) as pr_rank\n from\n pull_request_issues pri\n left join pull_requests pr on pri.pull_request_id = pr.id\n left join issues i on pri.issue_id = i.id\n where \n i.issue_key != 0\n),\n\n_final_results as(\n select\n distinct ccop.*, pi.type\n from \n _commit_count_of_pr ccop\n left join _pr_issues pi on ccop.pull_request_id = pi.pull_request_id\n where pi.pr_rank = 1\n order by 1\n)\n\nSELECT\n tag_name,\n case when type != '' then type else 'UNKNOWN' end as type,\n -- sum(pr_commit_count) as commit_count\n sum(pr_eloc) as dev_eq\nfrom _final_results\ngroup by 1,2", + "rawSql": "/* Get the work-type distribution in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 1 OFFSET 2), _combine_pr AS (SELECT pull_request_id AS id, commit_sha, p.pull_request_key AS pull_request_key FROM pull_request_commits LEFT JOIN pull_requests AS p ON pull_request_commits.pull_request_id = p.id WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) UNION SELECT id, merge_commit_sha, pull_request_key AS commit_sha FROM pull_requests WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))), _commit_count_of_pr AS (SELECT REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), 'tags/', 1)) AS tag_name, pr.id AS pull_request_id, COUNT(c.sha) AS pr_commit_count FROM refs_commits_diffs AS rcd LEFT JOIN commits AS c ON rcd.commit_sha = c.sha /* left join pull_request_commits prc on c.sha = prc.commit_sha */ LEFT JOIN _combine_pr AS pr ON c.sha = pr.commit_sha WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(rcd.new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND /* and rcd.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ REVERSE(SPLIT_PART(REVERSE(rcd.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1, 2), _pr_issues AS (SELECT pri.pull_request_id, pri.issue_id, i.issue_key, i.type, ROW_NUMBER() OVER (PARTITION BY issue_id ORDER BY pr.created_date ASC NULLS FIRST) AS pr_rank FROM pull_request_issues AS pri LEFT JOIN pull_requests AS pr ON pri.pull_request_id = pr.id LEFT JOIN issues AS i ON pri.issue_id = i.id WHERE i.issue_key <> '0'), _final_results AS (SELECT DISTINCT ccop.*, pi.type FROM _commit_count_of_pr AS ccop LEFT JOIN _pr_issues AS pi ON ccop.pull_request_id = pi.pull_request_id WHERE pi.pr_rank = 1 ORDER BY 1 NULLS FIRST) SELECT tag_name, CASE WHEN type <> '' THEN type ELSE 'UNKNOWN' END AS type, SUM(pr_commit_count) AS commit_count FROM _final_results GROUP BY 1, 2", "refId": "A", "select": [ [ @@ -2535,7 +2457,7 @@ "type": "piechart" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2568,9 +2490,10 @@ "h": 7, "w": 6, "x": 0, - "y": 70 + "y": 72 }, "id": 27, + "links": [], "options": { "colorMode": "value", "graphMode": "area", @@ -2586,7 +2509,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.6", + "pluginVersion": "13.0.2", "targets": [ { "format": "table", @@ -2594,7 +2517,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "-- Get each contributor's work in bugfixing in the last 5 tags\nwith refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n),\n\n_author_commits as(\n SELECT \n \tc.author_name,\n -- count(c.sha) as commit_count\n sum(c.dev_eq) total_dev_eq\n FROM \n refs_commits_diffs rcf\n -- left join commits c on rcf.commit_sha = c.sha\n left join commits c on rcf.commit_sha = c.dev_eq\n WHERE\n \t-- rcf.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n \tSUBSTRING_INDEX(rcf.new_ref_id,':', 3) in ($repo_id)\n \tand SUBSTRING_INDEX(rcf.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\n GROUP BY 1\n),\n\n_author_commits_running_total as(\n SELECT \n *, \n -- sum(commit_count) OVER (Order by commit_count desc) AS running_total\n sum(total_dev_eq) OVER (Order by total_dev_eq desc) AS running_total\n FROM \n _author_commits\n),\n\n_percentile as(\n SELECT \n author_name,\n -- commit_count,\n -- running_total/sum(commit_count) OVER () AS cumulative_percentage\n total_dev_eq,\n running_total/sum(total_dev_eq) OVER () AS cumulative_percentage\n FROM \n _author_commits_running_total\n)\n\n\nSELECT \n count(case when cumulative_percentage <= 0.8 then author_name else null end)/count(*) as \"contributors who contributed 80% of dev_eq\"\nFROM _percentile", + "rawSql": "/* Get each contributor's work in bugfixing in the last 5 tags */ WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5), _author_commits AS (SELECT c.author_name, COUNT(c.sha) AS commit_count FROM refs_commits_diffs AS rcf LEFT JOIN commits AS c ON rcf.commit_sha = c.sha WHERE SPLIT_PART(rcf.new_ref_id, ':', 4) /* rcf.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ IN (${repo_id}) AND REVERSE(SPLIT_PART(REVERSE(rcf.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1), _author_commits_running_total AS (SELECT *, SUM(commit_count) OVER (ORDER BY commit_count DESC NULLS LAST) AS running_total FROM _author_commits), _percentile AS (SELECT author_name, commit_count, CAST(running_total AS NUMERIC) / NULLIF(SUM(commit_count) OVER (), 0) AS cumulative_percentage FROM _author_commits_running_total) SELECT CAST(COUNT(CASE WHEN cumulative_percentage <= 0.8 THEN author_name ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"contributors who contributed 80% of dev_eq\" FROM _percentile", "refId": "A", "select": [ [ @@ -2622,7 +2545,7 @@ "type": "stat" }, { - "datasource": "mysql", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, "description": "", "fieldConfig": { "defaults": { @@ -2663,9 +2586,10 @@ "h": 7, "w": 18, "x": 6, - "y": 70 + "y": 72 }, "id": 3, + "links": [], "options": { "barWidth": 0.3, "groupWidth": 0.7, @@ -2688,7 +2612,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "with refs_commits_diffs as(\n SELECT\n new_refs.id as new_ref_id, old_refs.id as old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha\n FROM\n commits_diffs\n LEFT JOIN refs new_refs on new_refs.commit_sha = commits_diffs.new_commit_sha\n LEFT JOIN refs old_refs on old_refs.commit_sha = commits_diffs.old_commit_sha\n),\n\n_last_5_tags as(\n SELECT \n -- distinct new_ref_id, old_ref_id\n distinct SUBSTRING_INDEX(new_ref_id,':', -1) as new_ref_id, SUBSTRING_INDEX(old_ref_id,':', -1) as old_ref_id\n FROM \n refs_commits_diffs\n WHERE\n\t\tSUBSTRING_INDEX(new_ref_id,':', 3) in ($repo_id)\n\tORDER BY 1 desc\n\tLIMIT 5\n)\n\n\nSELECT \n\tc.author_name,\n -- count(c.sha) total_dev_eq\n sum(c.dev_eq) total_dev_eq\nFROM \n refs_commits_diffs rcf\n -- left join commits c on rcf.commit_sha = c.sha\n left join commits c on rcf.commit_sha = c.dev_eq\nWHERE\n\t-- rcf.new_ref_id in (SELECT new_ref_id FROM _last_5_tags)\n\tSUBSTRING_INDEX(rcf.new_ref_id,':', 3) in ($repo_id)\n\tand SUBSTRING_INDEX(rcf.new_ref_id,':', -1) in (SELECT new_ref_id FROM _last_5_tags)\nGROUP BY 1\nORDER BY 2 desc\nlimit 10", + "rawSql": "WITH refs_commits_diffs AS (SELECT new_refs.id AS new_ref_id, old_refs.id AS old_ref_id, commits_diffs.commit_sha, new_commit_sha, old_commit_sha FROM commits_diffs LEFT JOIN refs AS new_refs ON new_refs.commit_sha = commits_diffs.new_commit_sha LEFT JOIN refs AS old_refs ON old_refs.commit_sha = commits_diffs.old_commit_sha), _last_5_tags AS (SELECT DISTINCT /* distinct new_ref_id, old_ref_id */ REVERSE(SPLIT_PART(REVERSE(new_ref_id), ':', 1)) AS new_ref_id, REVERSE(SPLIT_PART(REVERSE(old_ref_id), ':', 1)) AS old_ref_id FROM refs_commits_diffs WHERE ('${repo_id:csv}' = '' OR SPLIT_PART(new_ref_id, ':', 4) = ANY(ARRAY[${repo_id:singlequote}]::text[])) ORDER BY 1 DESC NULLS LAST LIMIT 5) SELECT c.author_name, COUNT(c.sha) AS total_dev_eq FROM refs_commits_diffs AS rcf LEFT JOIN commits AS c ON rcf.commit_sha = c.sha WHERE SPLIT_PART(rcf.new_ref_id, ':', 4) /* rcf.new_ref_id in (SELECT new_ref_id FROM _last_5_tags) */ IN (${repo_id}) AND REVERSE(SPLIT_PART(REVERSE(rcf.new_ref_id), ':', 1)) IN (SELECT new_ref_id FROM _last_5_tags) GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 10", "refId": "A", "select": [ [ @@ -2714,12 +2638,36 @@ ], "title": "8.2 Top Contributors [Last 5 Tags]", "type": "barchart" + }, + { + "datasource": null, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 59, + "options": { + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" } ], "refresh": "", "schemaVersion": 30, "style": "dark", - "tags": [], + "tags": [ + "OSS Maintainer Dashboard" + ], "templating": { "list": [ { @@ -2733,8 +2681,8 @@ "$__all" ] }, - "datasource": "mysql", - "definition": "select concat(name, '--', id) as text from repos", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM repos", "description": null, "error": null, "hide": 0, @@ -2743,7 +2691,7 @@ "multi": true, "name": "repo_id", "options": [], - "query": "select concat(name, '--', id) as text from repos", + "query": "SELECT name || '--' || id AS text FROM repos", "refresh": 1, "regex": "/^(?.*)--(?.*)$/", "skipUrlSync": false, @@ -2757,8 +2705,8 @@ "to": "now" }, "timepicker": {}, - "timezone": "", - "title": "EE_GitHub_Release_Quality_and_Contribution_Analysis", - "uid": "2xuOaQUnk5", - "version": 4 + "timezone": "utc", + "title": "GitHub Release Quality and Contribution Analysis", + "uid": "2xuOaQUnk4-pg", + "version": 3 } \ No newline at end of file diff --git a/grafana/dashboards/postgresql/kiro-credits-dora.json b/grafana/dashboards/postgresql/kiro-credits-dora.json new file mode 100644 index 00000000000..551529fd01a --- /dev/null +++ b/grafana/dashboards/postgresql/kiro-credits-dora.json @@ -0,0 +1,709 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "DORA Dashboard", + "type": "link", + "url": "/d/qNo8_0M4z/dora" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage Dashboard", + "type": "link", + "url": "/d/qdev_user_report" + } + ], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Kiro Credits + DORA Correlation\nThis dashboard correlates **Kiro AI usage (credits and messages)** with **DORA** performance indicators at a weekly aggregate level.\n\n- **Pearson's r** measures linear correlation: negative r suggests higher AI usage may correlate with shorter cycle times.\n- Data is aggregated by **week** and joined on `week_start`.", + "mode": "markdown" + }, + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation coefficient between weekly Kiro credits and PR cycle time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "Need more data" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": -0.3 + }, + { + "color": "orange", + "value": 0.3 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 4 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_cycle_hours FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, p.avg_cycle_hours AS y FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", + "refId": "A" + } + ], + "title": "Credits vs Cycle Time (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total Kiro credits consumed in period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(credits_used) AS \"Credits Used\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total Credits", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Median PR cycle time in hours", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 4 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _ranked AS (SELECT CAST(pr_cycle_time AS NUMERIC) / NULLIF(60.0, 0) AS ct, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS prank FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date)) SELECT ROUND(MAX(ct), 1) AS \"Median Cycle Time\" FROM _ranked WHERE prank <= 0.5", + "refId": "A" + } + ], + "title": "Median Cycle Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Cycle time comparison: weeks with above-median vs below-median AI credits usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 6, + "options": { + "barRadius": 0.1, + "barWidth": 0.6, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _pr_weekly AS (SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS avg_ct FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits, p.avg_ct FROM _kiro_weekly AS k INNER JOIN _pr_weekly AS p ON k.week_start = p.week_start), _med AS (SELECT weekly_credits AS med FROM _joined ORDER BY weekly_credits NULLS FIRST LIMIT 1 OFFSET (SELECT FLOOR(CAST(COUNT(*) AS NUMERIC) / NULLIF(2, 0)) FROM _joined)) SELECT CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END AS \"Tier\", ROUND(CAST(AVG(j.avg_ct) AS DECIMAL), 1) AS \"Avg Cycle Time\" FROM _joined AS j, _med AS m GROUP BY CASE WHEN j.weekly_credits >= m.med THEN 'High AI Usage' ELSE 'Low AI Usage' END", + "refId": "A" + } + ], + "title": "Cycle Time: High vs Low AI Usage", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 10, + "panels": [], + "title": "Weekly Trends", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly Kiro credits consumed", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS \"Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly Kiro Credits & Active Users", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly average PR cycle time in hours", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' AS time, ROUND(CAST(CAST(AVG(pr_cycle_time) AS NUMERIC) / NULLIF(60.0, 0) AS DECIMAL), 1) AS \"Avg Cycle Time (hrs)\", COUNT(*) AS \"PRs Merged\" FROM project_pr_metrics WHERE NOT pr_merged_date IS NULL AND NOT pr_cycle_time IS NULL AND $__timeFilter(pr_merged_date) GROUP BY CAST(pr_merged_date AS DATE) - (EXTRACT(ISODOW FROM CAST(pr_merged_date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly PR Cycle Time & Volume", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 20, + "panels": [], + "title": "Deployment Frequency", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Pearson correlation between weekly credits and deployment count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "Need more data" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 20 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "WITH _kiro_weekly AS (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, SUM(credits_used) AS weekly_credits FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day'), _deploy_weekly AS (SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS week_start, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day'), _joined AS (SELECT k.weekly_credits AS x, d.deploys AS y FROM _kiro_weekly AS k INNER JOIN _deploy_weekly AS d ON k.week_start = d.week_start), _stats AS (SELECT COUNT(*) AS n, AVG(x) AS mx, AVG(y) AS my, STDDEV_POP(x) AS sx, STDDEV_POP(y) AS sy FROM _joined) SELECT CASE WHEN s.n < 4 THEN NULL WHEN s.sx = '0' OR s.sy = '0' THEN 0 ELSE ROUND(CAST((SELECT SUM((j.x - s.mx) * (j.y - s.my)) FROM _joined AS j) AS NUMERIC) / NULLIF((s.n * s.sx * s.sy), 0), 2) END AS \"r\" FROM _stats AS s", + "refId": "A" + } + ], + "title": "Credits vs Deploy Freq (r)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly credits overlaid with deployment count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 20 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(deploys) AS \"Deployments\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS deploys FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploys FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY CAST(cdc.finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(cdc.finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly Credits vs Deployments", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 30, + "panels": [], + "title": "Change Failure Rate", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Change failure rate: % of deployments that caused incidents", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(credits) AS \"Credits Used\", SUM(cfr) AS \"Change Failure Rate\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(credits_used) AS credits, 0 AS cfr FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS credits, CAST(SUM(CASE WHEN NOT i.id IS NULL THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(DISTINCT d.deployment_id), 0) AS cfr FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY cdc.cicd_deployment_id) AS d LEFT JOIN project_incident_deployment_relationships AS pidr ON d.deployment_id = pidr.deployment_id LEFT JOIN incidents AS i ON pidr.id = i.id GROUP BY CAST(d.deployment_finished_date AS DATE) - (EXTRACT(ISODOW FROM CAST(d.deployment_finished_date AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly Credits vs Change Failure Rate", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "dora", + "kiro", + "correlation" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Credits + DORA Correlation", + "uid": "kiro_credits_dora-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/language-ai-heatmap.json b/grafana/dashboards/postgresql/language-ai-heatmap.json new file mode 100644 index 00000000000..63aa23d65da --- /dev/null +++ b/grafana/dashboards/postgresql/language-ai-heatmap.json @@ -0,0 +1,276 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Completion requests, avg context size, and completion rate per language", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Requests" + } + ] + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"Language\", COUNT(*) AS \"Requests\", ROUND(CAST(AVG(completions_count) AS DECIMAL), 2) AS \"Avg Completions\", ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\", COUNT(DISTINCT user_id) AS \"Users\" FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Language Completion Profile", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily completion requests by language", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "lineWidth": 1, + "stacking": { + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS metric, COUNT(*) AS value FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Daily Completions by Language", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Which file types are most active during chat sessions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 3, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file)' ELSE active_file_extension END AS \"File Type\", COUNT(*) AS \"Chat Events\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"active_file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 10", + "refId": "A" + } + ], + "title": "Active File Types During Chat", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Context size trends for top languages", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, file_extension AS metric, ROUND(AVG(left_context_length + right_context_length)) AS value FROM _tool_q_dev_completion_log WHERE $__timeFilter(timestamp) AND file_extension IN (SELECT file_extension FROM _tool_q_dev_completion_log GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 5) GROUP BY CAST(timestamp AS DATE), \"file_extension\" ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Avg Context Size by Language (Top 5)", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "language", + "completions" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Language AI Heatmap", + "uid": "kiro_language_heatmap-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/multi-ai-comparison.json b/grafana/dashboards/postgresql/multi-ai-comparison.json new file mode 100644 index 00000000000..bca47881221 --- /dev/null +++ b/grafana/dashboards/postgresql/multi-ai-comparison.json @@ -0,0 +1,464 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Kiro Usage", + "type": "link", + "url": "/d/qdev_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Copilot Adoption", + "type": "link", + "url": "/d/copilot_adoption" + } + ], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Multi-AI Tool Comparison: Copilot vs Kiro\nCompare GitHub Copilot and Kiro (Amazon Q Developer) adoption and usage side by side.\n\n**Note:** Copilot metrics come from `_tool_copilot_enterprise_daily_metrics` (enterprise-level aggregates). Kiro metrics come from `_tool_q_dev_user_report` and `_tool_q_dev_user_data`. Select the Copilot connection and scope using the dropdowns above.", + "mode": "markdown" + }, + "title": "About", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "panels": [], + "title": "Active Users", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly active users for both tools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT user_id) AS \"Kiro Active Users\" FROM _tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, MAX(daily_active_users) AS \"Copilot Active Users\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "B" + } + ], + "title": "Weekly Active Users Comparison", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 10, + "panels": [], + "title": "Code Generation", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly code generation and acceptance events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(inline_suggestions_count) AS \"Kiro: Suggestions\", SUM(inline_acceptance_count) AS \"Kiro: Accepted\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Kiro: Code Suggestions & Acceptance", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly Copilot code generation and acceptance events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(code_generation_activity_count) AS \"Copilot: Suggestions\", SUM(code_acceptance_activity_count) AS \"Copilot: Accepted\" FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Copilot: Code Suggestions & Acceptance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 20, + "panels": [], + "title": "Lines of Code & Acceptance Rate", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly AI-generated lines of code accepted", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(kiro_loc) AS \"Kiro LOC Accepted\", SUM(copilot_loc) AS \"Copilot LOC Added\" FROM (SELECT CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' AS time, SUM(inline_ai_code_lines + chat_ai_code_lines) AS kiro_loc, 0 AS copilot_loc FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY CAST(date AS DATE) - (EXTRACT(ISODOW FROM CAST(date AS DATE)) - 1) * INTERVAL '1 day' UNION ALL SELECT CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day' AS time, 0 AS kiro_loc, SUM(loc_added_sum) AS copilot_loc FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day) GROUP BY CAST(day AS DATE) - (EXTRACT(ISODOW FROM CAST(day AS DATE)) - 1) * INTERVAL '1 day') AS combined GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "LOC Accepted: Kiro vs Copilot", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Overall acceptance rates for both tools", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 22, + "options": { + "barRadius": 0.1, + "barWidth": 0.5, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'Kiro' AS \"Tool\", ROUND(CAST(SUM(inline_acceptance_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0), 1) AS \"Acceptance Rate\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Copilot' AS \"Tool\", ROUND(CAST(SUM(code_acceptance_activity_count) * 100.0 AS NUMERIC) / NULLIF(NULLIF(SUM(code_generation_activity_count), 0), 0), 1) FROM _tool_copilot_enterprise_daily_metrics WHERE ('${copilot_connection_id}' = '' OR connection_id::text = '${copilot_connection_id}') AND scope_id = '${copilot_scope_id}' AND $__timeFilter(day)", + "refId": "A" + } + ], + "title": "Acceptance Rate Comparison", + "type": "bargauge" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "copilot", + "kiro", + "comparison" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC NULLS LAST", + "hide": 0, + "label": "Copilot Connection", + "name": "copilot_connection_id", + "options": [], + "query": "SELECT DISTINCT connection_id FROM _tool_copilot_scopes ORDER BY connection_id DESC NULLS LAST", + "refresh": 1, + "type": "query" + }, + { + "current": {}, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS BIGINT)", + "hide": 0, + "label": "Copilot Scope", + "name": "copilot_scope_id", + "options": [], + "query": "SELECT DISTINCT id FROM _tool_copilot_scopes WHERE connection_id = CAST('${copilot_connection_id}' AS BIGINT)", + "refresh": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Multi-AI Tool Comparison", + "uid": "multi_ai_comparison-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/pager-duty.json b/grafana/dashboards/postgresql/pager-duty.json new file mode 100644 index 00000000000..6469d4b2708 --- /dev/null +++ b/grafana/dashboards/postgresql/pager-duty.json @@ -0,0 +1,1295 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 20, + "links": [ + { + "asDropdown": false, + "icon": "bolt", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Homepage", + "tooltip": "", + "type": "link", + "url": "/grafana/d/Lv1XbLHnk/data-specific-dashboards-homepage" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "Data Source Specific Dashboard" + ], + "targetBlank": false, + "title": "Metric dashboards", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 13, + "x": 0, + "y": 0 + }, + "id": 128, + "links": [ + { + "targetBlank": true, + "title": "PagerDuty", + "url": "https://devlake.apache.org/docs/Plugins/pagerduty" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the incident data from PagerDuty.\n- Data Source Required: PagerDuty", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 126, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "1. Incident Resolution Status", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 114, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 116, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Resolved Incidents [Created in Selected Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. Total number of incidents created.\n2. The requirements being calculated are filtered by \"requirement creation time\" (time filter at the upper-right corner) and \"Jira board\" (\"Choose Board\" filter at the upper-left corner)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 4 + }, + "id": 131, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT b.name AS service, i.issue_key, i.description, i.original_status, i.priority, i.created_date, i.updated_date, ROUND(CAST((CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS DECIMAL), 1) AS lead_time_days, i.url FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of Incidents [Created in Selected Time Range]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 117, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT COUNT(DISTINCT i.id) AS total_count, COUNT(DISTINCT CASE WHEN i.original_status = 'resolved' THEN i.id ELSE NULL END) AS resolved_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT NOW() AS time, CAST(1.0 * resolved_count AS NUMERIC) / NULLIF(total_count, 0) AS requirement_delivery_rate FROM _requirements", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate [Incidents created in the selected time range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Resolution Rate(%)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 12, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 10 + }, + "id": 121, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.created_date AS DATE)) + 1) AS time, CAST(1.0 * COUNT(DISTINCT CASE WHEN i.original_status = 'resolved' THEN i.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS resolved_rate FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT time, resolved_rate FROM _requirements ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Incident Resolution Rate over Time [Incidents Created in Selected Time Range]", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 110, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "2. Mean Time to Resolve (MTTR)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "/^value$/", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "MTTR [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT i.lead_time_minutes, PERCENT_RANK() OVER (ORDER BY lead_time_minutes ASC NULLS FIRST) AS ranks FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))) SELECT MAX(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS value FROM _ranks WHERE ranks <= 0.8", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "80% Incidents' MTTR are less than # [Incidents Resolved in Select Time Range]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Incident Age(days)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 8, + "y": 17 + }, + "id": 17, + "interval": "", + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _requirements AS (SELECT CAST(i.resolution_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(i.resolution_date AS DATE)) + 1) AS time, AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS mean_incident_age FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, mean_incident_age FROM _requirements ORDER BY time ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Mean MTTR [Incidents Resolved in Select Time Range]", + "type": "barchart" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "1. The cumulative distribution of MTTR\n2. Each point refers to the percent rank of a distinct duration to resolve incidents.", + "fill": 0, + "fillGradient": 4, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 8, + "links": [ + { + "targetBlank": true, + "title": "Incident Age", + "url": "https://devlake.apache.org/docs/Metrics/IncidentAge" + } + ], + "nullPointMode": "null", + "options": { + "alertThreshold": false + }, + "percentage": false, + "pluginVersion": "13.0.2", + "pointradius": 0.5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _ranks AS (SELECT ROUND(CAST(i.lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) AS lead_time_day FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id WHERE i.original_status = 'resolved' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR bi.board_id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY lead_time_day ASC NULLS FIRST) SELECT NOW() AS time, LPAD(lead_time_day || 'd', 4, ' ') AS metric, PERCENT_RANK() OVER (ORDER BY lead_time_day ASC NULLS FIRST) AS value FROM _ranks ORDER BY lead_time_day ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "progress" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ca_analysis", + "timeColumn": "create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "thresholds": [ + { + "$$hashKey": "object:469", + "colorMode": "ok", + "fill": true, + "line": true, + "op": "lt", + "value": 0.8, + "yaxis": "right" + } + ], + "timeRegions": [], + "title": "Cumulative Distribution of MTTR [Incidents Resolved in Select Time Range]", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transformations": [], + "type": "graph", + "xaxis": { + "mode": "series", + "show": true, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "$$hashKey": "object:76", + "format": "percentunit", + "label": "Percent Rank (%)", + "logBase": 1, + "max": "1.2", + "show": true + }, + { + "$$hashKey": "object:77", + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 130, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'pagerduty%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "SELECT name || '--' || id FROM boards WHERE id::text LIKE 'pagerduty%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "PagerDuty", + "uid": "abb26d07-3268-4bdf-b871-6e39b37b9e00-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/q-dev-dora.json b/grafana/dashboards/postgresql/q-dev-dora.json new file mode 100644 index 00000000000..2a5368236d1 --- /dev/null +++ b/grafana/dashboards/postgresql/q-dev-dora.json @@ -0,0 +1,1233 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "DORA Dashboard", + "tooltip": "", + "type": "link", + "url": "/d/qNo8_0M4z/dora" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Q Dev Dashboard", + "tooltip": "", + "type": "link", + "url": "/d/qdev_user_data/q-dev-user-data-dashboard" + } + ], + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## AI-Powered DORA Dashboard\nThis dashboard correlates **Q Dev (AI coding assistant)** usage metrics with **DORA** performance indicators to help understand the impact of AI-assisted development on engineering efficiency.\n\n- **Left side**: Q Dev AI usage metrics (code generation, acceptance rate)\n- **Right side**: DORA metrics (Lead Time, Deployment Frequency, Change Failure Rate)\n- **Correlation charts**: Show trends over time to identify potential relationships", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "panels": [], + "title": "Overview Statistics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of unique users who used Q Dev AI features", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Q Dev Users\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Q Dev Active Users", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Total AI-generated code lines accepted (Inline + Chat)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 4 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(inline_ai_code_lines + chat_ai_code_lines) AS \"AI Accepted Lines\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Total AI Code Lines", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Acceptance rate of inline AI suggestions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.3 + }, + { + "color": "green", + "value": 0.5 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 4 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "AI Acceptance Rate", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of production deployments in selected period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 4 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT cdc.cicd_deployment_id) AS \"Deployments\" FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date)", + "refId": "A" + } + ], + "title": "Total Deployments", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Median lead time for changes in hours", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 24 + }, + { + "color": "red", + "value": 168 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 4 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _pr_stats AS (SELECT DISTINCT pr.id, ppm.pr_cycle_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date)), _median_ranks AS (SELECT *, PERCENT_RANK() OVER (ORDER BY pr_cycle_time NULLS FIRST) AS ranks FROM _pr_stats) SELECT ROUND(CAST(CAST(MAX(pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS DECIMAL), 1) AS \"Lead Time (hours)\" FROM _median_ranks WHERE ranks <= 0.5", + "refId": "A" + } + ], + "title": "Median Lead Time", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage of deployments that caused incidents", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 0.15 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _deployments AS (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()), _failure_caused AS (SELECT d.deployment_id, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM _deployments AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1) SELECT CASE WHEN COUNT(deployment_id) = 0 THEN NULL ELSE CAST(SUM(has_incident) AS NUMERIC) / NULLIF(COUNT(deployment_id), 0) END AS \"Change Failure Rate\" FROM _failure_caused", + "refId": "A" + } + ], + "title": "Change Failure Rate", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 9, + "panels": [], + "title": "AI Usage vs DORA Metrics Correlation", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Compare AI code generation trends with Lead Time for Changes. A negative correlation (AI lines up, Lead Time down) suggests AI is helping accelerate delivery.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "AI Accepted Lines" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "custom.axisLabel", + "value": "AI Code Lines" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Median Lead Time (hours)" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Lead Time (hours)" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH ai_monthly AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), lead_time_monthly AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM pull_requests AS pr JOIN project_pr_metrics AS ppm ON ppm.id = pr.id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.\"table\" = 'repos' JOIN cicd_deployment_commits AS cdc ON ppm.deployment_commit_id = cdc.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND NOT ppm.pr_cycle_time IS NULL AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, lt.month), '%Y-%m-%d') AS time, ai.ai_lines AS \"AI Accepted Lines\", ROUND(lt.avg_lead_time, 1) AS \"Median Lead Time (hours)\" FROM ai_monthly AS ai LEFT JOIN lead_time_monthly AS lt ON ai.month = lt.month WHERE NOT ai.month IS NULL OR NOT lt.month IS NULL ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "AI Code Generation vs Lead Time Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Compare AI suggestion acceptance rate with deployment frequency. Higher acceptance rate may indicate better AI integration and potentially more deployments.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "AI Acceptance Rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "custom.axisLabel", + "value": "Acceptance Rate" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Deployment Count" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Deployments" + }, + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH ai_acceptance AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), deployment_count AS (SELECT TO_CHAR(CAST(MAX(cdc.finished_date) AS TIMESTAMP), 'YYYY-MM-01') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deploy_count FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, dc.month), '%Y-%m-%d') AS time, ai.acceptance_rate AS \"AI Acceptance Rate\", dc.deploy_count AS \"Deployment Count\" FROM ai_acceptance AS ai LEFT JOIN deployment_count AS dc ON ai.month = dc.month WHERE NOT ai.month IS NULL OR NOT dc.month IS NULL ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "AI Acceptance Rate vs Deployment Frequency", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Compare AI-generated tests with Change Failure Rate. More AI-generated tests might correlate with lower failure rates.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "AI Generated Tests" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "custom.axisLabel", + "value": "Tests Generated" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Change Failure Rate" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Failure Rate" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH ai_tests AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') AS month, SUM(test_generation_generated_tests) AS generated_tests FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01')), cfr_monthly AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM-01')) SELECT TO_DATE(COALESCE(ai.month, cfr.month), '%Y-%m-%d') AS time, ai.generated_tests AS \"AI Generated Tests\", cfr.cfr AS \"Change Failure Rate\" FROM ai_tests AS ai LEFT JOIN cfr_monthly AS cfr ON ai.month = cfr.month WHERE NOT ai.month IS NULL OR NOT cfr.month IS NULL ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "AI Test Generation vs Change Failure Rate", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Compare active Q Dev users with Code Review findings. More AI-assisted code review might catch issues earlier.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active Users" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "custom.axisLabel", + "value": "Users" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Code Review Findings" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "Findings" + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT TO_DATE(TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01'), '%Y-%m-%d') AS time, COUNT(DISTINCT user_id) AS \"Active Users\", SUM(code_review_findings_count) AS \"Code Review Findings\" FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM-01') ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Q Dev Users vs Code Review Findings", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 14, + "panels": [], + "title": "Monthly Comparison Table", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Monthly summary comparing Q Dev AI metrics with DORA metrics side by side", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "AI Acceptance Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + }, + { + "id": "max", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Change Failure Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + }, + { + "id": "max", + "value": 0.3 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Lead Time (hours)" + }, + "properties": [ + { + "id": "unit", + "value": "h" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 15, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Month" + } + ] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH ai_metrics AS (SELECT TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT user_id) AS active_users, SUM(inline_ai_code_lines + chat_ai_code_lines) AS ai_lines, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS acceptance_rate, SUM(test_generation_generated_tests) AS generated_tests, SUM(code_review_findings_count) AS review_findings FROM _tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY TO_CHAR(CAST(date AS TIMESTAMP), 'YYYY-MM')), dora_metrics AS (SELECT TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM') AS month, COUNT(DISTINCT cdc.cicd_deployment_id) AS deployments, CAST(AVG(ppm.pr_cycle_time) AS NUMERIC) / NULLIF(60, 0) AS avg_lead_time FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' LEFT JOIN cicd_deployment_commits AS cdc2 ON cdc.cicd_deployment_id = cdc2.cicd_deployment_id LEFT JOIN project_pr_metrics AS ppm ON ppm.deployment_commit_id = cdc2.id WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' AND $__timeFilter(cdc.finished_date) GROUP BY TO_CHAR(CAST(cdc.finished_date AS TIMESTAMP), 'YYYY-MM')), cfr_metrics AS (SELECT TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM') AS month, CAST(SUM(has_incident) AS NUMERIC) / NULLIF(NULLIF(COUNT(deployment_id), 0), 0) AS cfr FROM (SELECT d.deployment_id, d.deployment_finished_date, COUNT(DISTINCT CASE WHEN NOT i.id IS NULL THEN d.deployment_id ELSE NULL END) AS has_incident FROM (SELECT cdc.cicd_deployment_id AS deployment_id, MAX(cdc.finished_date) AS deployment_finished_date FROM cicd_deployment_commits AS cdc JOIN project_mapping AS pm ON cdc.cicd_scope_id = pm.row_id AND pm.\"table\" = 'cicd_scopes' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND cdc.result = 'SUCCESS' AND cdc.environment = 'PRODUCTION' GROUP BY 1 HAVING MAX(cdc.finished_date) BETWEEN $__timeFrom() AND $__timeTo()) AS d LEFT JOIN project_incident_deployment_relationships AS pim ON d.deployment_id = pim.deployment_id LEFT JOIN incidents AS i ON pim.id = i.id GROUP BY 1, 2) AS failure_data GROUP BY TO_CHAR(CAST(deployment_finished_date AS TIMESTAMP), 'YYYY-MM')) SELECT COALESCE(ai.month, dm.month, cfr.month) AS \"Month\", COALESCE(ai.active_users, 0) AS \"Q Dev Users\", COALESCE(ai.ai_lines, 0) AS \"AI Code Lines\", ai.acceptance_rate AS \"AI Acceptance Rate\", COALESCE(ai.generated_tests, 0) AS \"AI Tests\", COALESCE(ai.review_findings, 0) AS \"Review Findings\", COALESCE(dm.deployments, 0) AS \"Deployments\", ROUND(dm.avg_lead_time, 1) AS \"Lead Time (hours)\", cfr.cfr AS \"Change Failure Rate\" FROM ai_metrics AS ai LEFT JOIN dora_metrics AS dm ON ai.month = dm.month LEFT JOIN cfr_metrics AS cfr ON ai.month = cfr.month ORDER BY ai.month DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Monthly Q Dev vs DORA Metrics Comparison", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 39, + "tags": [ + "q_dev", + "DORA", + "AI", + "correlation" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT name FROM projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "SELECT DISTINCT name FROM projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Q Dev + DORA Correlation", + "uid": "qdev_dora_correlation-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/qdev_executive.json b/grafana/dashboards/postgresql/qdev_executive.json new file mode 100644 index 00000000000..8ea7bbb8c8f --- /dev/null +++ b/grafana/dashboards/postgresql/qdev_executive.json @@ -0,0 +1,768 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Usage (New)", + "tooltip": "Kiro Usage Dashboard - Credits & Messages (new format)", + "type": "link", + "url": "/d/qdev_user_report" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Feature Metrics (Legacy)", + "tooltip": "Kiro Legacy Feature Metrics (old format)", + "type": "link", + "url": "/d/qdev_feature_metrics" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "Prompt Logging", + "tooltip": "Kiro AI Activity Insights - Prompt Logging", + "type": "link", + "url": "/d/qdev_logging" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "KPI Overview (cross-source)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distinct users with chat activity in the last 7 days (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"WAU\" FROM lake._tool_q_dev_chat_log WHERE timestamp >= NOW() - INTERVAL '7 DAY'", + "refId": "A" + } + ], + "title": "Weekly Active Users (logging)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average credits spent per accepted line of code (new report + legacy metrics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(NULLIF(SUM(d.total_accepted), 0), 0), 2) AS \"Credits per Accepted Line\" FROM (SELECT user_id, date, SUM(credits_used) AS credits_used FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r JOIN (SELECT user_id, \"date\", (inline_ai_code_lines + chat_ai_code_lines + code_fix_accepted_lines + dev_accepted_lines) AS total_accepted FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)) AS d ON r.user_id = d.user_id AND r.date = d.date", + "refId": "A" + } + ], + "title": "Credits Efficiency (new + legacy)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage of inline suggestions accepted (from legacy feature metrics)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) AS \"Acceptance %\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Inline Acceptance Rate (legacy)", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Percentage of users who used steering rules (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) * 100 AS DECIMAL), 0) AS \"Steering %\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Adoption (logging)", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 101, + "panels": [], + "title": "User Engagement (logging data: _tool_q_dev_chat_log)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly active user count over time (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, COUNT(DISTINCT user_id) AS \"Active Users\" FROM (SELECT user_id, (EXTRACT(ISOYEAR FROM timestamp) * 100 + EXTRACT(WEEK FROM timestamp))::int AS yw FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS t GROUP BY \"yw\" ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly Active Users Trend", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "New vs returning users by week (from prompt logging)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT TO_DATE(yw || ' Monday', 'IYYYIW FMDay') AS time, SUM(CASE WHEN yw = first_yw THEN 1 ELSE 0 END) AS \"New Users\", SUM(CASE WHEN yw <> first_yw THEN 1 ELSE 0 END) AS \"Returning Users\" FROM (SELECT DISTINCT u.user_id, (EXTRACT(ISOYEAR FROM u.timestamp) * 100 + EXTRACT(WEEK FROM u.timestamp))::int AS yw, f.first_yw FROM lake._tool_q_dev_chat_log AS u JOIN (SELECT user_id, (EXTRACT(ISOYEAR FROM MIN(timestamp)) * 100 + EXTRACT(WEEK FROM MIN(timestamp)))::int AS first_yw FROM lake._tool_q_dev_chat_log GROUP BY user_id) AS f ON u.user_id = f.user_id WHERE $__timeFilter(u.timestamp)) AS weekly GROUP BY yw ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "New vs Returning Users (Weekly)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 102, + "panels": [], + "title": "Credits & Subscription (new format: _tool_q_dev_user_report)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Cumulative credits this month vs projected total (from new user_report)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(SUM(credits_used)) OVER (ORDER BY date NULLS FIRST) AS \"Cumulative Credits\", (SELECT CAST(SUM(credits_used) AS NUMERIC) / NULLIF(COUNT(DISTINCT date), 0) * EXTRACT(DAY FROM CAST(CAST(DATE_TRUNC('MONTH', CURRENT_DATE) + INTERVAL '1 MONTH' - INTERVAL '1 DAY' AS DATE) AS DATE)) FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE))) AS \"Projected Monthly\" FROM lake._tool_q_dev_user_report WHERE YEAR(CAST(date AS DATE)) = YEAR(CAST(CURRENT_DATE AS DATE)) AND MONTH(CAST(date AS DATE)) = MONTH(CAST(CURRENT_DATE AS DATE)) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Credits Pace vs Projected (This Month)", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Power tier users with no activity in the last 14 days (from new user_report)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 12, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", MAX(subscription_tier) AS \"Tier\", ROUND(SUM(credits_used), 1) AS \"Total Credits Used\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND subscription_tier = 'POWER' GROUP BY \"user_id\" HAVING MAX(date) < NOW() - INTERVAL '14 DAY' ORDER BY MAX(date) NULLS FIRST", + "refId": "A" + } + ], + "title": "Idle Power Users (No Activity in 14 Days)", + "type": "table" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 103, + "panels": [], + "title": "Cross-Source: User Productivity (new report + legacy metrics)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Per-user productivity combining credits (new format) with feature metrics (legacy). Only shows users present in both data sources.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 11, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(d.display_name), d.user_id) AS \"User\", COALESCE(MAX(r.subscription_tier), '') AS \"Tier\", ROUND(SUM(r.credits_used), 1) AS \"Credits Used\", SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) AS \"Total Accepted Lines\", CASE WHEN SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines) > 0 THEN ROUND(CAST(SUM(r.credits_used) AS NUMERIC) / NULLIF(SUM(d.chat_ai_code_lines + d.inline_ai_code_lines + d.code_fix_accepted_lines + d.dev_accepted_lines), 0), 2) ELSE NULL END AS \"Credits/Line\", ROUND(CAST(SUM(d.inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(d.inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept Rate\", SUM(d.code_review_findings_count) AS \"Review Findings\", SUM(d.test_generation_event_count) AS \"Test Gen Events\", SUM(d.dev_accepted_lines) AS \"Agentic Lines\", MIN(d.date) AS \"First Active\", MAX(d.date) AS \"Last Active\" FROM lake._tool_q_dev_user_data AS d LEFT JOIN (SELECT user_id, date, SUM(credits_used) AS credits_used, MAX(subscription_tier) AS subscription_tier FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", date) AS r ON d.user_id = r.user_id AND d.date = r.date WHERE $__timeFilter(d.date) GROUP BY d.user_id ORDER BY SUM(r.credits_used) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "User Productivity & Efficiency", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "executive", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Executive Dashboard", + "uid": "qdev_executive-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/qdev_feature_metrics.json b/grafana/dashboards/postgresql/qdev_feature_metrics.json new file mode 100644 index 00000000000..3ea698aa550 --- /dev/null +++ b/grafana/dashboards/postgresql/qdev_feature_metrics.json @@ -0,0 +1,948 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "High-level summary of legacy feature-level activity metrics (from by_user_analytic CSV reports)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(inline_suggestions_count) AS \"Inline Suggestions\", SUM(inline_acceptance_count) AS \"Inline Accepted\", SUM(chat_messages_sent) AS \"Chat Messages\", SUM(chat_ai_code_lines) AS \"Chat AI Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Agentic Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Legacy Feature Metrics Overview", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 20, + "panels": [], + "title": "Inline Suggestions", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily inline suggestion and acceptance counts", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", SUM(inline_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Inline Suggestions & Acceptance", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Acceptance rates for inline suggestions, code fix, and inline chat over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Acceptance Rate Trends", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 21, + "panels": [], + "title": "Chat & Agentic (Dev)", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily chat messages sent and AI-generated code lines from chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Messages Sent\", SUM(chat_messages_interacted) AS \"Messages Interacted\", SUM(chat_ai_code_lines) AS \"AI Code Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Chat Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Agentic (Dev) code generation and acceptance metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Generation Events\", SUM(dev_generated_lines) AS \"Generated Lines\", SUM(dev_accepted_lines) AS \"Accepted Lines\", SUM(dev_acceptance_event_count) AS \"Acceptance Events\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Agentic (Dev) Activity", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 22, + "panels": [], + "title": "Code Review, Test Gen & Transformations", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code review findings and test generation metrics over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(code_review_findings_count) AS \"Review Findings\", SUM(code_review_succeeded_event_count) AS \"Reviews Succeeded\", SUM(code_review_failed_event_count) AS \"Reviews Failed\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Code Review Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Test generation events and acceptance over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(test_generation_generated_tests) AS \"Tests Generated\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(test_generation_generated_lines) AS \"Lines Generated\", SUM(test_generation_accepted_lines) AS \"Lines Accepted\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Test Generation Activity", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Doc generation and code transformation events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(doc_generation_accepted_line_additions) AS \"Doc Lines Accepted\", SUM(transformation_event_count) AS \"Transformation Events\", SUM(transformation_lines_generated) AS \"Transform Lines Generated\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Doc Generation & Transformations", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Number of users who used each feature in the selected period", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 9, + "options": { + "barRadius": 0.1, + "barWidth": 0.8, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'Chat' AS \"Feature\", COUNT(DISTINCT CASE WHEN chat_messages_sent > 0 THEN user_id END) AS \"Users\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Inline Suggestions', COUNT(DISTINCT CASE WHEN inline_suggestions_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Fix', COUNT(DISTINCT CASE WHEN code_fix_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Code Review', COUNT(DISTINCT CASE WHEN code_review_succeeded_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Doc Generation', COUNT(DISTINCT CASE WHEN doc_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Test Generation', COUNT(DISTINCT CASE WHEN test_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Dev (Agentic)', COUNT(DISTINCT CASE WHEN dev_generation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) UNION ALL SELECT 'Transformation', COUNT(DISTINCT CASE WHEN transformation_event_count > 0 THEN user_id END) FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Feature Adoption (Users per Feature)", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 23, + "panels": [], + "title": "Per-User Detail", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Per-user breakdown of legacy feature-level metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(inline_suggestions_count) AS \"Suggestions\", SUM(inline_acceptance_count) AS \"Accepted\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 1) || '%' AS \"Accept %\", SUM(chat_messages_sent) AS \"Chat Msgs\", SUM(chat_ai_code_lines) AS \"Chat Lines\", SUM(dev_accepted_lines) AS \"Agentic Lines\", SUM(code_review_findings_count) AS \"Review Findings\", SUM(test_generation_accepted_tests) AS \"Tests Accepted\", SUM(doc_generation_event_count) AS \"Doc Gen\", SUM(transformation_event_count) AS \"Transforms\", MIN(date) AS \"First Active\", MAX(date) AS \"Last Active\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_suggestions_count) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Per-User Feature Metrics", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "legacy", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Legacy Feature Metrics", + "uid": "qdev_feature_metrics-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/qdev_logging.json b/grafana/dashboards/postgresql/qdev_logging.json new file mode 100644 index 00000000000..032b4fcfa16 --- /dev/null +++ b/grafana/dashboards/postgresql/qdev_logging.json @@ -0,0 +1,1118 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Overview of logging event metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT (SELECT COUNT(*) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Chat Users\", (SELECT COUNT(DISTINCT conversation_id) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) AND conversation_id <> '') AS \"Conversations\", (SELECT COUNT(*) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Events\", (SELECT COUNT(DISTINCT user_id) FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp)) AS \"Completion Users\", (SELECT SUM(code_reference_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Code References\", (SELECT SUM(web_link_count) FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)) AS \"Web Links Cited\"", + "refId": "A" + } + ], + "title": "Logging Overview", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Hourly distribution of AI usage activity (chat + completions)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Hour of Day", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.8, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT LPAD(CAST(hour_of_day AS TEXT), 2, '0') AS \"Hour\", SUM(chat_count) AS \"Chat Events\", SUM(completion_count) AS \"Completion Events\" FROM (SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, COUNT(*) AS chat_count, 0 AS completion_count FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp) UNION ALL SELECT EXTRACT(HOUR FROM timestamp) AS hour_of_day, 0 AS chat_count, COUNT(*) AS completion_count FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY EXTRACT(HOUR FROM timestamp)) AS combined GROUP BY hour_of_day ORDER BY hour_of_day NULLS FIRST", + "refId": "A" + } + ], + "title": "Active Hours Distribution", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of chat trigger types: MANUAL (chat window) vs INLINE_CHAT", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 14 + }, + "id": 11, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN chat_trigger_type = '' OR chat_trigger_type IS NULL THEN '(unknown)' ELSE chat_trigger_type END AS \"Trigger Type\", COUNT(*) AS \"Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"chat_trigger_type\" ORDER BY COUNT(*) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Chat Trigger Type Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of model usage across chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 14 + }, + "id": 3, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^Requests$/", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN model_id = '' OR model_id IS NULL THEN '(unknown)' ELSE model_id END AS \"Model\", COUNT(*) AS \"Requests\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"model_id\" ORDER BY COUNT(*) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Model Usage Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Top file extensions used with inline completions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN file_extension = '' THEN '(unknown)' ELSE file_extension END AS \"File Type\", COUNT(*) AS \"Completions\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY \"file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "refId": "A" + } + ], + "title": "File Type Usage (Completions)", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average number of chat events per conversation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, CAST(COUNT(*) AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END), 0), 0) AS \"Avg Turns per Conversation\", COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS \"Unique Conversations\", COUNT(*) AS \"Total Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "refId": "A" + } + ], + "title": "Conversation Depth Analysis", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily chat and completion events over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT time, SUM(chat) AS \"Chat Events\", SUM(completions) AS \"Completion Events\" FROM (SELECT CAST(timestamp AS DATE) AS time, COUNT(*) AS chat, 0 AS completions FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) UNION ALL SELECT CAST(timestamp AS DATE) AS time, 0 AS chat, COUNT(*) AS completions FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE)) AS combined GROUP BY time ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Daily Event Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Per-user logging activity summary", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(u.display_name, u.user_id) AS \"User\", u.user_id AS \"User ID\", u.chat_events AS \"Chat Events\", u.conversations AS \"Conversations\", ROUND(CAST(u.chat_events AS NUMERIC) / NULLIF(NULLIF(u.conversations, 0), 0), 1) AS \"Avg Turns\", COALESCE(c.completion_events, 0) AS \"Completion Events\", COALESCE(c.files_count, 0) AS \"Distinct Files\", ROUND(u.avg_prompt_len) AS \"Avg Prompt Len\", ROUND(u.avg_response_len) AS \"Avg Response Len\", u.steering_count AS \"Steering Uses\", u.spec_count AS \"Spec Mode Uses\", u.code_ref_count AS \"Code Refs\", u.web_link_count AS \"Web Links\", u.models_used AS \"Models Used\", u.first_seen AS \"First Seen\", GREATEST(u.last_seen, COALESCE(c.last_seen, u.last_seen)) AS \"Last Seen\" FROM (SELECT user_id, MAX(display_name) AS display_name, COUNT(*) AS chat_events, COUNT(DISTINCT CASE WHEN conversation_id <> '' THEN conversation_id END) AS conversations, AVG(prompt_length) AS avg_prompt_len, AVG(response_length) AS avg_response_len, STRING_AGG(DISTINCT CASE WHEN model_id <> '' AND NOT model_id IS NULL THEN model_id END, ', ' ORDER BY model_id NULLS FIRST) AS models_used, SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS steering_count, SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS spec_count, SUM(code_reference_count) AS code_ref_count, SUM(web_link_count) AS web_link_count, MIN(timestamp) AS first_seen, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS u LEFT JOIN (SELECT user_id, COUNT(*) AS completion_events, COUNT(DISTINCT file_name) AS files_count, MAX(timestamp) AS last_seen FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY user_id) AS c ON u.user_id = c.user_id ORDER BY u.user_id NULLS FIRST", + "refId": "A" + } + ], + "title": "Per-User Activity", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of Kiro feature adoption: Steering, Spec Mode, and Plain Chat", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 48 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Using Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Using Spec Mode\", SUM(CASE WHEN has_steering = FALSE AND is_spec_mode = FALSE THEN 1 ELSE 0 END) AS \"Plain Chat\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Kiro Feature Adoption", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Top file extensions active during chat events", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 48 + }, + "id": 9, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT CASE WHEN active_file_extension = '' OR active_file_extension IS NULL THEN '(no file active)' ELSE active_file_extension END AS \"File Type\", COUNT(*) AS \"Chat Events\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"active_file_extension\" ORDER BY COUNT(*) DESC NULLS LAST LIMIT 15", + "refId": "A" + } + ], + "title": "Active File Types in Chat", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "How often Kiro responses include code references and web links", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 48 + }, + "id": 12, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(CASE WHEN code_reference_count > 0 THEN 1 ELSE 0 END) AS \"With Code References\", SUM(CASE WHEN web_link_count > 0 THEN 1 ELSE 0 END) AS \"With Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"With Followup Prompts\", SUM(CASE WHEN code_reference_count = '0' AND web_link_count = '0' AND has_followup_prompts = FALSE THEN 1 ELSE 0 END) AS \"Plain Response\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Response Enrichment Breakdown", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average and maximum prompt/response lengths over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, AVG(prompt_length) AS \"Avg Prompt Length\", AVG(response_length) AS \"Avg Response Length\", MAX(prompt_length) AS \"Max Prompt Length\", MAX(response_length) AS \"Max Response Length\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "refId": "A" + } + ], + "title": "Prompt & Response Length Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Average code context size provided to inline completions over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Characters", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 64 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, ROUND(AVG(left_context_length)) AS \"Avg Left Context\", ROUND(AVG(right_context_length)) AS \"Avg Right Context\", ROUND(AVG(left_context_length + right_context_length)) AS \"Avg Total Context\" FROM lake._tool_q_dev_completion_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "refId": "A" + } + ], + "title": "Completion Context Size Trends", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily trend of code references and web links in chat responses", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) AS time, SUM(code_reference_count) AS \"Code References\", SUM(web_link_count) AS \"Web Links\", SUM(CASE WHEN has_followup_prompts = TRUE THEN 1 ELSE 0 END) AS \"Followup Prompts\" FROM lake._tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) ORDER BY CAST(timestamp AS DATE) NULLS FIRST", + "refId": "A" + } + ], + "title": "Response Enrichment Trends", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "logging", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro AI Activity Insights", + "uid": "qdev_logging-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/qdev_user_data.json b/grafana/dashboards/postgresql/qdev_user_data.json new file mode 100644 index 00000000000..d446fca81db --- /dev/null +++ b/grafana/dashboards/postgresql/qdev_user_data.json @@ -0,0 +1,1182 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 100, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Overview of key user metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT user_id) AS \"Active Users\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Acceptance Rate (Inline Suggestion)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Acceptance Rate (Code Fix)\", SUM(transformation_lines_ingested) AS \"Ingested Lines (Java Transform)\", SUM(transformation_lines_generated) AS \"Generated Lines (Java Transform)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Lines (Inline Chat)\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Acceptance Rate (Inline Chat)\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date)", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Overall Usage Statistics", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily AI code line changes across all users", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(chat_ai_code_lines) AS \"Chat Accepted Lines\", SUM(code_fix_accepted_lines) AS \"Code Fix Accepted Lines\", SUM(code_fix_generated_lines) AS \"Code Fix Generated Lines\", SUM(transformation_lines_ingested) AS \"Java Transform Ingested Lines\", SUM(transformation_lines_generated) AS \"Java Transform Generated Lines\", SUM(inline_ai_code_lines) AS \"Inline Suggestion Accepted Lines\", SUM(inline_chat_accepted_line_additions) AS \"Inline Chat Accepted Line Additions\", SUM(inline_chat_accepted_line_deletions) AS \"Inline Chat Accepted Line Deletions\", SUM(inline_chat_dismissed_line_additions) AS \"Inline Chat Dismissed Line Additions\", SUM(inline_chat_dismissed_line_deletions) AS \"Inline Chat Dismissed Line Deletions\", SUM(inline_chat_rejected_line_additions) AS \"Inline Chat Rejected Line Additions\", SUM(inline_chat_rejected_line_deletions) AS \"Inline Chat Rejected Line Deletions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Daily AI Code Line Changes", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily AI interaction trends across all users", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(chat_messages_sent) AS \"Chat Messages Sent\", SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(transformation_event_count) AS \"Java Transform Event Count\", SUM(inline_acceptance_count) AS \"Inline Suggestion Accepted Suggestions\", SUM(inline_suggestions_count) AS \"Inline Suggestion Count\", SUM(inline_chat_total_event_count) AS \"Inline Chat Total Suggestions\", SUM(inline_chat_acceptance_event_count) AS \"Inline Chat Accepted Suggestions\", SUM(inline_chat_dismissal_event_count) AS \"Inline Chat Dismissed Suggestions\", SUM(inline_chat_rejection_event_count) AS \"Inline Chat Rejected Suggestions\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Daily AI Interactions", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Code review metrics over time", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(code_fix_acceptance_event_count) AS \"Code Fix Accepted Event Count\", SUM(code_fix_generation_event_count) AS \"Code Fix Generated Event Count\", SUM(code_review_findings_count) AS \"Total Findings\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Code Review Metrics", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily acceptance rate of AI suggestions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) AS \"Code Fix\", CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) AS \"Inline Suggestions\", CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) AS \"Inline Chat\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Daily AI Suggestion Acceptance Rate", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "User AI interaction metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Acceptance Rate" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inline Chat Accepted Events" + }, + "properties": [ + { + "id": "custom.width", + "value": 239 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Acceptance Rate (Inline Suggestion)" + }, + "properties": [ + { + "id": "custom.width", + "value": 172 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", SUM(chat_ai_code_lines) AS \"Accepted Lines (Chat)\", SUM(transformation_lines_ingested) AS \"Lines Ingested (Java Transform)\", SUM(transformation_lines_generated) AS \"Lines Generated (Java Transform)\", SUM(transformation_event_count) AS \"Event Count (Java Transform)\", SUM(code_review_findings_count) AS \"Findings (Code Review)\", SUM(code_fix_accepted_lines) AS \"Accepted Lines (Code Fix)\", SUM(code_fix_generated_lines) AS \"Generated Lines (Code Fix)\", SUM(code_fix_acceptance_event_count) AS \"Accepted Count (Code Fix)\", SUM(code_fix_generation_event_count) AS \"Generated Count (Code Fix)\", ROUND(CAST(SUM(code_fix_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(code_fix_generation_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Code Fix)\", SUM(inline_ai_code_lines) AS \"Accepted Lines (Inline Suggestion)\", SUM(inline_acceptance_count) AS \"Accepted Count (Inline Suggestion)\", SUM(inline_suggestions_count) AS \"Total Count (Inline Suggestion)\", ROUND(CAST(SUM(inline_acceptance_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_suggestions_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Suggestion)\", SUM(inline_chat_accepted_line_additions) AS \"Accepted Line Additions (Inline Chat)\", SUM(inline_chat_accepted_line_deletions) AS \"Accepted Line Deletions (Inline Chat)\", SUM(inline_chat_acceptance_event_count) AS \"Accepted Events (Inline Chat)\", SUM(inline_chat_total_event_count) AS \"Total Events (Inline Chat)\", ROUND(CAST(SUM(inline_chat_acceptance_event_count) AS NUMERIC) / NULLIF(NULLIF(SUM(inline_chat_total_event_count), 0), 0) * 100, 2) || '%' AS \"Acceptance Rate (Inline Chat)\", SUM(doc_generation_event_count) AS \"Doc Gen Events\", SUM(test_generation_event_count) AS \"Test Gen Events\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"user_id\" ORDER BY SUM(inline_ai_code_lines) DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "User Interactions", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily doc generation events and accepted/rejected lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(doc_generation_event_count) AS \"Doc Generation Events\", SUM(doc_generation_accepted_line_additions) AS \"Accepted Line Additions\", SUM(doc_generation_accepted_line_updates) AS \"Accepted Line Updates\", SUM(doc_generation_rejected_line_additions) AS \"Rejected Line Additions\", SUM(doc_generation_rejected_line_updates) AS \"Rejected Line Updates\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Doc Generation Metrics", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily test generation events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(test_generation_event_count) AS \"Test Generation Events\", SUM(test_generation_accepted_tests) AS \"Accepted Tests\", SUM(test_generation_generated_tests) AS \"Generated Tests\", SUM(test_generation_accepted_lines) AS \"Accepted Lines\", SUM(test_generation_generated_lines) AS \"Generated Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Test Generation Metrics", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily agentic dev events and lines", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "group": [], + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(dev_generation_event_count) AS \"Dev Generation Events\", SUM(dev_acceptance_event_count) AS \"Dev Acceptance Events\", SUM(dev_generated_lines) AS \"Dev Generated Lines\", SUM(dev_accepted_lines) AS \"Dev Accepted Lines\" FROM lake._tool_q_dev_user_data WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "value" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "timeColumn": "time", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Dev (Agentic) Metrics", + "type": "timeseries" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "user_data" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Code Metrics Dashboard", + "uid": "qdev_user_data-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/qdev_user_report.json b/grafana/dashboards/postgresql/qdev_user_report.json new file mode 100644 index 00000000000..592d8a75daf --- /dev/null +++ b/grafana/dashboards/postgresql/qdev_user_report.json @@ -0,0 +1,464 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 101, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Overview of credits and usage metrics", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(credits_used) AS \"Total Credits Used\", COUNT(DISTINCT user_id) AS \"Active Users\", SUM(total_messages) AS \"Total Messages\", SUM(chat_conversations) AS \"Total Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date)", + "refId": "A" + } + ], + "title": "Overview Stats", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily credits consumed broken down by subscription tier", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, subscription_tier AS metric, SUM(credits_used) AS value FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\", \"subscription_tier\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Daily Credits Consumed by Subscription Tier", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Daily messages and conversations broken down by client type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "max", + "sum" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT date AS time, SUM(CASE WHEN client_type = 'KIRO_IDE' THEN total_messages ELSE 0 END) AS \"Messages (IDE)\", SUM(CASE WHEN client_type = 'KIRO_CLI' THEN total_messages ELSE 0 END) AS \"Messages (CLI)\", SUM(CASE WHEN client_type = 'PLUGIN' THEN total_messages ELSE 0 END) AS \"Messages (Plugin)\", SUM(chat_conversations) AS \"Conversations\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"date\" ORDER BY date NULLS FIRST", + "refId": "A" + } + ], + "title": "Daily Messages & Conversations", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Distribution of users across subscription tiers", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 4, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT subscription_tier AS \"Tier\", COUNT(DISTINCT user_id) AS \"Users\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) AND NOT subscription_tier IS NULL AND subscription_tier <> '' GROUP BY \"subscription_tier\" ORDER BY COUNT(DISTINCT user_id) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Subscription Tier Distribution", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Per-user credits, messages, and subscription details", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Credits Used" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Overage" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "Yes": { + "color": "orange", + "index": 0 + }, + "No": { + "color": "green", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 5, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", subscription_tier AS \"Tier\", client_type AS \"Client\", SUM(credits_used) AS \"Credits Used\", SUM(total_messages) AS \"Messages\", SUM(chat_conversations) AS \"Conversations\", SUM(overage_credits_used) AS \"Overage Credits\", CASE WHEN MAX(CAST(overage_enabled AS BIGINT)) = 1 THEN 'Yes' ELSE 'No' END AS \"Overage\", MIN(date) AS \"First Activity\", MAX(date) AS \"Last Activity\" FROM lake._tool_q_dev_user_report WHERE $__timeFilter(date) GROUP BY \"user_id\", \"subscription_tier\", \"client_type\" ORDER BY user_id DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Per-User Credits & Activity", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "user_report", + "kiro" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-30d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Usage Dashboard", + "uid": "qdev_user_report-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/sonar-qube-cloud.json b/grafana/dashboards/postgresql/sonar-qube-cloud.json new file mode 100644 index 00000000000..496fd63a8fd --- /dev/null +++ b/grafana/dashboards/postgresql/sonar-qube-cloud.json @@ -0,0 +1,1768 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 40, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "links": [ + { + "targetBlank": true, + "title": "SonarQube", + "url": "https://devlake.apache.org/docs/Plugins/sonarqube" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the code quality metrics from SonarCloud.\n- Data Source Required: SonarCloud\n- Snapshot panels (Software Quality, Security Review, Test, etc.) show the latest scan and do not honor the time filter.\n- **Historical Trends** panels at the bottom respond to the time range selector and show metric history over time.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 16, + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Software Quality", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 3, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT ci.id) FROM cq_issues AS ci JOIN cq_issue_impacts AS cii ON ci.id = cii.cq_issue_id WHERE ('${project_id:csv}' = '' OR ci.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND cii.software_quality = 'SECURITY' AND ('${severity:csv}' = '' OR ci.severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Security", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT ci.id) FROM cq_issues AS ci JOIN cq_issue_impacts AS cii ON ci.id = cii.cq_issue_id WHERE ('${project_id:csv}' = '' OR ci.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND cii.software_quality = 'RELIABILITY' AND ('${severity:csv}' = '' OR ci.severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Reliability", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 20, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "vertical", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT ci.id) FROM cq_issues AS ci JOIN cq_issue_impacts AS cii ON ci.id = cii.cq_issue_id WHERE ('${project_id:csv}' = '' OR ci.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND cii.software_quality = 'MAINTAINABILITY' AND ('${severity:csv}' = '' OR ci.severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Maintainability", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 21, + "panels": [], + "title": "Security Review", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Security Hotspots\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'HOTSPOTS' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Security Hotspots", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 13, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(CASE WHEN status <> 'TO_REVIEW' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT id), 0) * 100 AS DECIMAL), 2) || '%' AS \"Reviewed\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'HOTSPOTS' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hotspots Reviewed", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 12, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Test & Maintainability", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 17 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Test", + "url": "https://devlake.apache.org/docs/Metrics/CQTest" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST((SUM(lines_to_cover) - SUM(uncovered_lines)) AS NUMERIC) / NULLIF(SUM(lines_to_cover), 0) * 100, 1) || '% ' || 'Coverage on ' || ROUND(CAST(CAST(SUM(lines_to_cover) AS NUMERIC) / NULLIF(1000, 0) AS DECIMAL), 0) || 'k Lines to cover' FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Test Coverage", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "code smells" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 17 + }, + "id": 14, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Maintainability-Debt", + "url": "https://devlake.apache.org/docs/Metrics/CQMaintainability-Debt" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT id) AS \"Code Smells\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'CODE_SMELL' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Maintainability - Code Smells", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 17 + }, + "id": 7, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Maintainability-Debt", + "url": "https://devlake.apache.org/docs/Metrics/CQMaintainability-Debt" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT FLOOR(CAST(CAST(SUM(debt) AS NUMERIC) / NULLIF(8, 0) AS NUMERIC) / NULLIF(60, 0)) || ' day(s) ' || FLOOR(CAST((SUM(debt) % 480) AS NUMERIC) / NULLIF(60, 0)) || ' hour(s) ' AS \"Debt\" FROM cq_issues WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND type = 'CODE_SMELL' AND ('${severity:csv}' = '' OR severity::text = ANY(ARRAY[${severity:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Maintainability - Debt", + "type": "stat" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 6, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Duplications", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Blocks", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedBlocks" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(duplicated_blocks) FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duplicated Blocks", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 23 + }, + "id": 9, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Lines", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedLines" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(SUM(duplicated_lines) AS NUMERIC) / NULLIF(SUM(num_of_lines), 0) * 100, 1) || '% ' || 'Duplications on ' || ROUND(CAST(CAST(SUM(ncloc) AS NUMERIC) / NULLIF(1000, 0) AS DECIMAL), 0) || 'k Lines' FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Duplicated Lines", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 22, + "panels": [], + "title": "Size", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 29 + }, + "id": 23, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Blocks", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedBlocks" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(ncloc) FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Lines of Code", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 29 + }, + "id": 24, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Blocks", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedBlocks" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT SUM(num_of_lines) FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Lines", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 29 + }, + "id": 25, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Duplicated Blocks", + "url": "https://devlake.apache.org/docs/Metrics/CQDuplicatedBlocks" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT file_path) FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[]))", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Files", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 26, + "panels": [], + "title": "Overall Code Quality Metrics", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "color-text" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 17, + "links": [ + { + "targetBlank": true, + "title": "Code Quality Issue Count", + "url": "https://devlake.apache.org/docs/Metrics/CQIssueCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "dataset": "lake", + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT file_name, num_of_lines AS \"Lines of Code\", bugs AS \"Bugs\", vulnerabilities AS \"Vulnerabilities\", code_smells AS \"Code Smells\", security_hotspots AS \"Security Hotspots\", ROUND(coverage, 2) || '%' AS \"Coverage\", ROUND(duplicated_lines_density, 2) || '%' AS \"Duplications\" FROM cq_file_metrics WHERE ('${project_id:csv}' = '' OR project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) ORDER BY bugs DESC NULLS LAST LIMIT 20", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Code Quality Metrics by Files (Top 20 order by Bugs)", + "type": "table" + }, + { + "collapsed": true, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 27, + "panels": [ + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.coverage AS \"Coverage %\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Coverage Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.bugs AS \"Bugs\", pmh.vulnerabilities AS \"Vulnerabilities\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Bugs & Vulnerabilities Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.code_smells AS \"Code Smells\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Code Smells Trend", + "type": "timeseries" + }, + { + "datasource": "postgresql", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "lineWidth": 2, + "fillOpacity": 10, + "spanNulls": true, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "pointSize": 5 + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": "postgresql", + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT pmh.analysis_date AS time, pmh.duplicated_lines_density AS \"Duplication %\" FROM cq_project_metrics_history AS pmh WHERE ('${project_id:csv}' = '' OR pmh.project_key::text = ANY(ARRAY[${project_id:singlequote}]::text[])) AND $__timeFilter(pmh.analysis_date) ORDER BY pmh.analysis_date ASC NULLS FIRST", + "refId": "A" + } + ], + "title": "Duplication Trend", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Historical Trends", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Data Source Dashboard", + "Stable Data Sources" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id AS text FROM cq_projects", + "hide": 0, + "includeAll": true, + "label": "SonarQube Project", + "multi": true, + "name": "project_id", + "options": [], + "query": "SELECT name || '--' || id AS text FROM cq_projects", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT severity FROM cq_issues WHERE id::text LIKE 'sonar%'", + "hide": 0, + "includeAll": true, + "label": "Severity", + "multi": true, + "name": "severity", + "options": [], + "query": "SELECT DISTINCT severity FROM cq_issues WHERE id::text LIKE 'sonar%'", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "SonarQube Cloud", + "uid": "WA0qbuJ4l-pg", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/steering-adoption-tracker.json b/grafana/dashboards/postgresql/steering-adoption-tracker.json new file mode 100644 index 00000000000..f332a314483 --- /dev/null +++ b/grafana/dashboards/postgresql/steering-adoption-tracker.json @@ -0,0 +1,353 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN has_steering = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Steering", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(COUNT(DISTINCT CASE WHEN is_spec_mode = TRUE THEN user_id END) * 100.0 AS NUMERIC) / NULLIF(NULLIF(COUNT(DISTINCT user_id), 0), 0) AS DECIMAL), 0) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Users with Spec Mode", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT ROUND(CAST(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS DECIMAL), 1) AS \"value\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Requests with Steering", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Weekly trend of steering and spec mode adoption rate", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' AS time, CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Steering Rate\", CAST(SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(*), 0) AS \"Spec Mode Rate\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY CAST(timestamp AS DATE) - (EXTRACT(ISODOW FROM CAST(timestamp AS DATE)) - 1) * INTERVAL '1 day' ORDER BY time NULLS FIRST", + "refId": "A" + } + ], + "title": "Weekly Steering & Spec Mode Adoption Rate", + "type": "timeseries" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Compare prompt and response quality between steering and non-steering sessions", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "options": { + "barRadius": 0.1, + "barWidth": 0.5, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT 'With Steering' AS \"Mode\", ROUND(AVG(prompt_length)) AS \"Avg Prompt\", ROUND(AVG(response_length)) AS \"Avg Response\" FROM _tool_q_dev_chat_log WHERE has_steering = TRUE AND $__timeFilter(timestamp) UNION ALL SELECT 'Without Steering', ROUND(AVG(prompt_length)), ROUND(AVG(response_length)) FROM _tool_q_dev_chat_log WHERE has_steering = FALSE AND $__timeFilter(timestamp)", + "refId": "A" + } + ], + "title": "Steering Impact: Prompt & Response Length", + "type": "bargauge" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Per-user steering and spec mode usage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COALESCE(MAX(display_name), user_id) AS \"User\", COUNT(*) AS \"Total Chats\", SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) AS \"Steering\", SUM(CASE WHEN is_spec_mode = TRUE THEN 1 ELSE 0 END) AS \"Spec Mode\", ROUND(CAST(CAST(SUM(CASE WHEN has_steering = TRUE THEN 1 ELSE 0 END) * 100.0 AS NUMERIC) / NULLIF(COUNT(*), 0) AS DECIMAL), 1) AS \"Steering %\" FROM _tool_q_dev_chat_log WHERE $__timeFilter(timestamp) GROUP BY \"user_id\" ORDER BY COUNT(*) DESC NULLS LAST", + "refId": "A" + } + ], + "title": "Per-User Feature Adoption", + "type": "table" + } + ], + "preload": false, + "refresh": "5m", + "schemaVersion": 41, + "tags": [ + "q_dev", + "kiro", + "steering", + "adoption" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Kiro Steering & Spec Mode Adoption", + "uid": "kiro_steering_adoption-pg", + "version": 1 +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/weekly-bug-retro.json b/grafana/dashboards/postgresql/weekly-bug-retro.json new file mode 100644 index 00000000000..459baa7d6ec --- /dev/null +++ b/grafana/dashboards/postgresql/weekly-bug-retro.json @@ -0,0 +1,2274 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 49, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard can be used to track bugs with metrics such as [Bug Age](https://devlake.apache.org/docs/Metrics/BugAge).\n- Data Source Required: One of the issue tracking tools, e.g. [GitHub](https://devlake.apache.org/docs/UserManuals/ConfigUI/GitHub#step-3---adding-transformation-rules-optional), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab) or [Jira](https://devlake.apache.org/docs/UserManuals/ConfigUI/Jira#step-3---adding-transformation-rules-optional) (Scope Config required to define which issues are bugs).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 32, + "panels": [], + "title": "Overall Bug Count", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 46, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.71, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH bugs AS (SELECT CAST(i.created_date AS DATE) + -(EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT i.id) AS bug_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY \"time\" ORDER BY time DESC NULLS LAST), calendar_date AS (SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS d FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') >= ($__timeTo()::timestamp - INTERVAL '6 MONTH')), calendar_weeks AS (SELECT DISTINCT CAST(CAST(d AS DATE) + -(EXTRACT(ISODOW FROM CAST(d AS DATE)) - 1) * INTERVAL '1 day' AS DATE) AS start_of_week FROM calendar_date ORDER BY 1 ASC NULLS FIRST) SELECT TO_CHAR(CAST(cw.start_of_week AS TIMESTAMP), 'MM/DD') || ' - ' || TO_CHAR(CAST(cw.start_of_week + INTERVAL '7 DAY' AS TIMESTAMP), 'MM/DD') AS week, CASE WHEN NOT bug_count IS NULL THEN bug_count ELSE 0 END AS \"Weekly New Bugs\" FROM calendar_weeks AS cw LEFT JOIN bugs AS b ON cw.start_of_week = b.time ORDER BY cw.start_of_week ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Weekly New Bugs [Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 46, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 20, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.71, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH bugs AS (SELECT CAST(i.resolution_date AS DATE) + -(EXTRACT(ISODOW FROM CAST(i.resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT i.id) AS bug_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY \"time\" ORDER BY time DESC NULLS LAST), calendar_date AS (SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS d FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') >= ($__timeTo()::timestamp - INTERVAL '6 MONTH')), calendar_weeks AS (SELECT DISTINCT CAST(CAST(d AS DATE) + -(EXTRACT(ISODOW FROM CAST(d AS DATE)) - 1) * INTERVAL '1 day' AS DATE) AS start_of_week FROM calendar_date ORDER BY 1 ASC NULLS FIRST) SELECT TO_CHAR(CAST(cw.start_of_week AS TIMESTAMP), 'MM/DD') || ' - ' || TO_CHAR(CAST(cw.start_of_week + INTERVAL '7 DAY' AS TIMESTAMP), 'MM/DD') AS week, CASE WHEN NOT bug_count IS NULL THEN bug_count ELSE 0 END AS \"Weekly Closed Bugs\" FROM calendar_weeks AS cw LEFT JOIN bugs AS b ON cw.start_of_week = b.time ORDER BY cw.start_of_week ASC NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Weekly Closed Bugs [Selected Time Range]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 46, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 21, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.71, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH calendar_date AS (SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS d FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') >= ($__timeTo()::timestamp - INTERVAL '6 MONTH')), calendar_weeks AS (SELECT DISTINCT CAST(CAST(d AS DATE) + -(EXTRACT(ISODOW FROM CAST(d AS DATE)) - 1) * INTERVAL '1 day' AS DATE) AS start_of_week FROM calendar_date ORDER BY 1 ASC NULLS FIRST), created_bugs AS (SELECT CAST(i.created_date AS DATE) + -(EXTRACT(ISODOW FROM CAST(i.created_date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT i.id) AS bug_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND $__timeFilter(i.created_date) AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY \"time\" ORDER BY time DESC NULLS LAST), resolved_bugs AS (SELECT CAST(i.resolution_date AS DATE) + -(EXTRACT(ISODOW FROM CAST(i.resolution_date AS DATE)) - 1) * INTERVAL '1 day' AS time, COUNT(DISTINCT i.id) AS bug_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND $__timeFilter(i.resolution_date) AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) GROUP BY \"time\" ORDER BY time DESC NULLS LAST), weekly_new_bug AS (SELECT cw.start_of_week AS week, CASE WHEN NOT bug_count IS NULL THEN bug_count ELSE 0 END AS weekly_new_bug FROM calendar_weeks AS cw LEFT JOIN created_bugs AS cb ON cw.start_of_week = cb.time), weekly_closed_bug AS (SELECT cw.start_of_week AS week, CASE WHEN NOT bug_count IS NULL THEN bug_count ELSE 0 END AS weekly_closed_bug FROM calendar_weeks AS cw LEFT JOIN resolved_bugs AS cb ON cw.start_of_week = cb.time), weekly_updates AS (SELECT t1.week, weekly_new_bug, weekly_closed_bug FROM weekly_new_bug AS t1 LEFT JOIN weekly_closed_bug AS t2 ON t1.week = t2.week UNION SELECT t1.week, weekly_new_bug, weekly_closed_bug FROM weekly_new_bug AS t1 RIGHT JOIN weekly_closed_bug AS t2 ON t1.week = t2.week), original_open_bugs AS (SELECT COUNT(DISTINCT i.id) AS original_open_bug_count FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND i.created_date < $__timeFrom() AND (i.status <> 'DONE' OR $__timeFilter(i.resolution_date)) AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))), weekly_updated_without_null AS (SELECT week, COALESCE(weekly_new_bug, 0) AS weekly_new_bug, COALESCE(weekly_closed_bug, 0) AS weekly_closed_bug, original_open_bug_count FROM weekly_updates, original_open_bugs WHERE NOT week IS NULL), weekly_delta AS (SELECT *, (weekly_new_bug - weekly_closed_bug) AS weekly_delta FROM weekly_updated_without_null ORDER BY week ASC NULLS FIRST), final_data AS (SELECT *, TO_CHAR(CAST(week AS TIMESTAMP), 'MM/DD') || ' - ' || TO_CHAR(CAST(week + INTERVAL '7 DAY' AS TIMESTAMP), 'MM/DD') AS _week, SUM(weekly_delta) OVER (ORDER BY week ASC NULLS FIRST) AS weekly_accumulated FROM weekly_delta) SELECT _week, (original_open_bug_count + weekly_accumulated) AS \"Total No. of Outstanding Bugs By the End of Week\" FROM final_data", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Outstanding Bug Count by Week [Selected Time Range]", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 33, + "panels": [], + "title": "Detailed Bug Info", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 34 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.created_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day'", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Week's New Bug Count", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 34 + }, + "id": 26, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT priority, COUNT(DISTINCT i.id) AS \"Issue Number\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.created_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day' GROUP BY 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Week's New Bug Count by Priority", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Issue Number" + }, + "properties": [ + { + "id": "custom.width", + "value": 124 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Title" + }, + "properties": [ + { + "id": "custom.width", + "value": 640 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields.Url}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 4, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.issue_key AS \"Issue Number\", i.title AS \"Title\", i.url AS \"Url\", i.creator_name AS \"Creator\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.created_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day'", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Week's New Bugs", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 8, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.resolution_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day'", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Week's Fixed Bugs Count", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 14 + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 24, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0)) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.resolution_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day'", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Lead Time of Last Week's Resolved Bugs in Days", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Issue Number" + }, + "properties": [ + { + "id": "custom.width", + "value": 125 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Title" + }, + "properties": [ + { + "id": "custom.width", + "value": 639 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields.Url}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Lead Time in Days" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 10, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.issue_key AS \"Issue Number\", i.title AS \"Title\", CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS \"Lead Time in Days\", i.url AS \"Url\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.resolution_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day'", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Last Week's Fixed Bugs", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 23, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.3, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "horizontal", + "showValue": "auto", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT '#' || i.issue_key || ' ' || i.title AS issue_key, CAST(lead_time_minutes AS NUMERIC) / NULLIF(1440, 0) AS lead_time FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND status = 'DONE' AND i.status <> 'REJECTED' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND i.resolution_date BETWEEN TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - INTERVAL '1 DAY' * ((EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) + 7) AND TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0)) - (EXTRACT(ISODOW FROM TO_TIMESTAMP(CAST($__to AS NUMERIC) / NULLIF(1000, 0))) - 1) * INTERVAL '1 day' ORDER BY lead_time DESC NULLS LAST LIMIT 10", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top 10 Issue Lead Time [Last Week's Resolved Bugs in Days]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 58 + }, + "id": 16, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND NOT i.status IN ('DONE', 'REJECTED') AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Total Number of Outstanding Bugs", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 58 + }, + "id": 25, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0)) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND NOT i.status IN ('DONE', 'REJECTED') AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Age of Outstanding Bugs in Days", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 30 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Issue Number" + }, + "properties": [ + { + "id": "custom.width", + "value": 121 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Title" + }, + "properties": [ + { + "id": "custom.width", + "value": 641 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields.Url}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queue Time in Days" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 104 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Severity" + }, + "properties": [ + { + "id": "custom.width", + "value": 112 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 14, + "links": [ + { + "targetBlank": true, + "title": "Bug Count per 1k Lines of Code", + "url": "https://devlake.apache.org/docs/Metrics/BugCountPer1kLinesOfCode" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.issue_key AS \"Issue Number\", i.title AS \"Title\", priority AS \"Priority\", severity AS \"Severity\", CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS \"Queue Time in Days\", i.url AS \"Url\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND NOT i.status IN ('DONE', 'REJECTED') AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) AND ('${priority:csv}' = '' OR priority::text = ANY(ARRAY[${priority:singlequote}]::text[])) ORDER BY 'Queue Time' DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Outstanding Bugs Sort by Queue Time [All History]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Queue Time in Days", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 21 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 67 + }, + "id": 18, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.3, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "always", + "stacking": "none", + "text": {}, + "tooltip": { + "maxHeight": 600, + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 100 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT '#' || i.issue_key AS issue_key, CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS \"Queue Time in Days\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND NOT i.status IN ('DONE', 'REJECTED') AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY 2 DESC NULLS LAST LIMIT 100", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top 100 Bug Queue Time in Days [Outstanding Bugs]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 7 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Issue Number" + }, + "properties": [ + { + "id": "custom.width", + "value": 121 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Title" + }, + "properties": [ + { + "id": "custom.width", + "value": 641 + }, + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "${__data.fields.Url}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Queue Time in Days" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Url" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Priority" + }, + "properties": [ + { + "id": "custom.width", + "value": 103 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Severity" + }, + "properties": [ + { + "id": "custom.width", + "value": 112 + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 67 + }, + "id": 31, + "links": [ + { + "targetBlank": true, + "title": "Bug Age", + "url": "https://devlake.apache.org/docs/Metrics/BugAge" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT i.issue_key AS \"Issue Number\", i.title AS \"Title\", priority AS \"Priority\", severity AS \"Severity\", CAST(((EXTRACT(EPOCH FROM (NOW() - i.created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS \"Queue Time in Days\", i.url AS \"Url\" FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND NOT i.status IN ('DONE', 'REJECTED') AND i.assignee_name = '' AND ('${board_id:csv}' = '' OR b.id::text = ANY(ARRAY[${board_id:singlequote}]::text[])) ORDER BY 'Queue Time' DESC NULLS LAST", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Bugs to be Triaged Sort by Queue Time [All History]", + "type": "table" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 76 + }, + "id": 30, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/UserManuals/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Engineering Leads Dashboard", + "Highlights" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '-' || id AS text FROM boards", + "hide": 0, + "includeAll": true, + "label": "Board", + "multi": false, + "name": "board_id", + "options": [], + "query": "SELECT name || '-' || id AS text FROM boards", + "refresh": 1, + "regex": "/^(?.*)-(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "BUG", + "value": "BUG" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "issue_type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT priority FROM issues", + "hide": 0, + "includeAll": true, + "label": "Priority", + "multi": true, + "name": "priority", + "options": [], + "query": "SELECT DISTINCT priority FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Weekly Bug Retro", + "uid": "-5EKA5w7k-pg", + "version": 5, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/weekly-community-retro.json b/grafana/dashboards/postgresql/weekly-community-retro.json new file mode 100644 index 00000000000..54d392abf79 --- /dev/null +++ b/grafana/dashboards/postgresql/weekly-community-retro.json @@ -0,0 +1,2447 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 14, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 48, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard can be used to track community growth by OSS maintainers.\n- Data Source Required: GitHub users' organizations are used to filter issues/PRs created by certain users. [Publicize users' org](https://docs.github.com/en/account-and-profile/setting-up-and-managing-your-personal-account-on-github/managing-your-membership-in-organizations/publicizing-or-hiding-organization-membership) so that Apache DevLake can collect users' org.\n- This dashboard **DOES NOT** honor the time filter on the top-right corner.", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 42, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Community Issues", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 6, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "New Issue Count [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 8, + "y": 5 + }, + "id": 22, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT i.id) FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "New Issue Created by Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 14, + "y": 5 + }, + "id": 23, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(SUM(CASE WHEN NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[]))) THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(DISTINCT i.id), 0) AS community_issue_ratio FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Ratio of New Issue Created by the Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 45, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT DISTINCT i.creator_name FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_name IN (SELECT DISTINCT creator_name FROM issues WHERE created_date < CURRENT_DATE - INTERVAL '1 day' AND NOT creator_name IS NULL)", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "1st Time Issue Reporter [Previous Week]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 24, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT i.id AS issue_id, i.url, i.title, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))) SELECT 1 - CAST(COUNT(DISTINCT CASE WHEN comment_id IS NULL THEN issue_id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT issue_id), 0) AS response_rate FROM issue_comment_list", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Community Issues' Response Rate [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 449 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "url" + }, + "properties": [ + { + "id": "custom.width", + "value": 415 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "issue_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 117 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "queue_time_in_days" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "issue_key" + }, + "properties": [ + { + "id": "custom.width", + "value": 95 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 13 + }, + "id": 25, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT i.id AS issue_id, i.url, i.issue_key, i.title, i.creator_name, i.created_date AS issue_created_date, i.status, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))) SELECT issue_key, title, creator_name, issue_created_date, status, CAST(((EXTRACT(EPOCH FROM (NOW() - issue_created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS \"queue_time_in_days\", url FROM issue_comment_list WHERE comment_id IS NULL", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of New Community Issues with no Comments [Previous Week]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 21 + }, + "id": 26, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT i.id AS issue_id, i.url, i.title, i.created_date AS issue_created_date, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))) SELECT AVG(CAST(((EXTRACT(EPOCH FROM (comment_date - issue_created_date))/60)) AS NUMERIC) / NULLIF(1440, 0)) FROM issue_comment_list WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "New Community Issues' Average Response Time in Days [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 451 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "creator_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 146 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "issue_created_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 162 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "comment_date" + }, + "properties": [ + { + "id": "custom.width", + "value": 160 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "issue_key" + }, + "properties": [ + { + "id": "custom.width", + "value": 96 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "response_time_in_days" + }, + "properties": [ + { + "id": "custom.width", + "value": 211 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 21 + }, + "id": 28, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "WITH issue_comment_list AS (SELECT REVERSE(SPLIT_PART(REVERSE(i.url), '/', 1)) AS issue_number, i.url, i.issue_key, i.title, i.creator_name, i.created_date AS issue_created_date, ic.id AS comment_id, ic.created_date AS comment_date, ic.body, CASE WHEN NOT ic.id IS NULL THEN RANK() OVER (PARTITION BY i.id ORDER BY ic.created_date ASC NULLS FIRST) ELSE NULL END AS comment_rank FROM issues AS i JOIN board_issues AS bi ON i.id = bi.issue_id JOIN boards AS b ON bi.board_id = b.id LEFT JOIN issue_comments AS ic ON i.id = ic.issue_id WHERE ('${issue_type:csv}' = '' OR i.type::text = ANY(ARRAY[${issue_type:singlequote}]::text[])) AND CAST(i.created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR b.id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT i.creator_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))) SELECT issue_key, title, creator_name /* body, */, issue_created_date, comment_date, CAST(((EXTRACT(EPOCH FROM (comment_date - issue_created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS response_time_in_days, url FROM issue_comment_list WHERE comment_rank = 1", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of New Community Issues' 1st Comment [Previous Week]", + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 44, + "panels": [], + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "title": "Community PRs", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 30 + }, + "id": 31, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of New Pull Requests [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 30 + }, + "id": 34, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of New Pull Requests Created by the Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 30 + }, + "id": 35, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT CAST(SUM(CASE WHEN NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[]))) THEN 1 ELSE 0 END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS community_pr_ratio FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[]))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Ratio of New PR Created by the Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 38 + }, + "id": 36, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Number of Merged Pull Requests Created by the Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 7 + } + ] + }, + "unit": "d" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 38 + }, + "id": 39, + "links": [ + { + "targetBlank": true, + "title": "PR Time To Merge", + "url": "https://devlake.apache.org/docs/Metrics/PRTimeToMerge" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT AVG(CAST((EXTRACT(EPOCH FROM (merged_date - created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) FROM pull_requests WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Average Time to Merge of Community Pull Requests in Days [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "Contributors: the PR authors who have merged PR(s)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 38 + }, + "id": 46, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT DISTINCT author_name FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND NOT merged_date IS NULL AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT author_name IN (SELECT DISTINCT author_name FROM pull_requests WHERE created_date < CURRENT_DATE - INTERVAL '1 day' AND NOT author_name IS NULL)", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "First time Contributor [Previous Week]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red" + }, + { + "color": "#EAB839", + "value": 0.5 + }, + { + "color": "green", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 46 + }, + "id": 37, + "links": [ + { + "targetBlank": true, + "title": "PR Merge Rate", + "url": "https://devlake.apache.org/docs/Metrics/PRMergeRate" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT CAST(COUNT(DISTINCT CASE WHEN NOT merged_date IS NULL THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS merged_pull_request_ratio FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Ratio of Merged Pull Requests Created by the Community [Previous Week]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 7 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "title" + }, + "properties": [ + { + "id": "custom.width", + "value": 451 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "queue_time_in_days" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 16, + "x": 8, + "y": 46 + }, + "id": 51, + "links": [ + { + "targetBlank": true, + "title": "Requirement Count", + "url": "https://devlake.apache.org/docs/Metrics/RequirementCount" + } + ], + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "metricColumn": "none", + "queryType": "randomWalk", + "rawQuery": true, + "rawSql": "SELECT pull_request_key, title, status, author_name, created_date, CAST(((EXTRACT(EPOCH FROM (CURRENT_DATE - created_date))/60)) AS NUMERIC) / NULLIF(1440, 0) AS queue_time_in_days, url FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND merged_date IS NULL AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[])))", + "refId": "A", + "select": [ + [ + { + "params": [ + "script_version" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "_devlake_migration_history", + "timeColumn": "created_at", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "List of New Community Issues' 1st Comment [Previous Week]", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Merged PR Count", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 54, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 10, + "x": 0, + "y": 54 + }, + "id": 33, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + }, + "xTickLabelRotation": 45, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT author_name, COUNT(DISTINCT pr.id) AS pull_request_count FROM pull_requests AS pr WHERE CAST(created_date AS DATE) BETWEEN CURRENT_DATE - INTERVAL '1 day' AND CURRENT_DATE - (EXTRACT(ISODOW FROM CURRENT_DATE) - 1) * INTERVAL '1 day' AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL AND NOT author_id IN (SELECT DISTINCT id FROM accounts WHERE ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[]))) GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top 20 Community Contributors By Number of PRs Merged [Previous Week]", + "type": "barchart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 10, + "y": 54 + }, + "id": 52, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT COUNT(DISTINCT author_id) AS all_contributor_count FROM pull_requests AS pr WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL /* and author_id not in (select distinct id from accounts where ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[]))) */", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top Community Contributors By PRs Created [All History]", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 54 + }, + "id": 40, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "first" + ], + "fields": "", + "values": true + }, + "showUnfilled": true, + "text": { + "valueSize": 1 + }, + "valueMode": "color" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "SELECT author_name, COUNT(DISTINCT pr.id) AS merged_pull_request_count FROM pull_requests AS pr WHERE ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) AND NOT merged_date IS NULL /* and author_id not in (select distinct id from accounts where ('${org:csv}' = '' OR organization::text = ANY(ARRAY[${org:singlequote}]::text[]))) */ GROUP BY 1 ORDER BY 2 DESC NULLS LAST LIMIT 20", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Top 20 Community Contributors By Merged PR Numbers [All History]", + "type": "bargauge" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 50, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "OSS Maintainer Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '-' || id AS text FROM repos", + "hide": 0, + "includeAll": true, + "label": "Repo", + "multi": true, + "name": "repo_id", + "options": [], + "query": "SELECT name || '-' || id AS text FROM repos", + "refresh": 1, + "regex": "/^(?.*)-(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT type FROM issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "issue_type", + "options": [], + "query": "SELECT DISTINCT type FROM issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT DISTINCT organization FROM accounts WHERE organization <> ''", + "description": "", + "hide": 0, + "includeAll": true, + "label": "Community Definition - Issues and PRs NOT from Organization(s)", + "multi": true, + "name": "org", + "options": [], + "query": "SELECT DISTINCT organization FROM accounts WHERE organization <> ''", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Weekly Community Retro", + "uid": "VTr6Y_q7z-pg", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/dashboards/postgresql/work-logs.json b/grafana/dashboards/postgresql/work-logs.json new file mode 100644 index 00000000000..3ea68c79847 --- /dev/null +++ b/grafana/dashboards/postgresql/work-logs.json @@ -0,0 +1,1467 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 39, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "- Use Cases: This dashboard shows the work logs of a team or a developer\n- Data Sources Required to show all data: \n - One of the Git tools, e.g. [GitHub](https://devlake.apache.org/docs/Configuration/GitHub), [GitLab](https://devlake.apache.org/docs/Configuration/GitLab), [Bitbucket](https://devlake.apache.org/docs/Configuration/BitBucket) or [Azure DevOps](https://devlake.apache.org/docs/Configuration/AzureDevOps)\n - One of the issue tracking tools, e.g. [Jira](https://devlake.apache.org/docs/Configuration/Jira)\n - You also need to complete the [team configuration](https://devlake.apache.org/docs/Configuration/TeamConfiguration) to use this dashboard", + "mode": "markdown" + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Dashboard Introduction", + "type": "text" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 9, + "title": "Worklogs", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "Comment on PR": { + "color": "super-light-orange", + "index": 5 + }, + "Create an issue": { + "color": "light-blue", + "index": 0 + }, + "Finish a commit": { + "color": "light-purple", + "index": 2 + }, + "Issue resolved": { + "color": "semi-dark-blue", + "index": 1 + }, + "Open a PR": { + "color": "super-light-green", + "index": 3 + }, + "PR gets merged": { + "color": "green", + "index": 4 + }, + "Review PR": { + "color": "orange", + "index": 6 + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Activity" + }, + "properties": [ + { + "id": "custom.filterable", + "value": true + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-background" + } + }, + { + "id": "custom.width", + "value": 180 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Date" + }, + "properties": [ + { + "id": "custom.width", + "value": 214 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 228 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Details" + }, + "properties": [ + { + "id": "custom.width", + "value": 991 + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _activities AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY \"Date\" ORDER BY \"Time\" DESC NULLS LAST) AS _row_number FROM (SELECT TO_CHAR(CAST(created_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", created_date AS \"Time\", 'Create an issue' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.creator_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT TO_CHAR(CAST(resolution_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", resolution_date AS \"Time\", 'Issue resolved' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.assignee_id = a.account_id WHERE $__timeFilter(resolution_date) UNION SELECT TO_CHAR(CAST(authored_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", authored_date AS \"Time\", 'Finish a commit' AS \"Activity\", message || ' #' || sha AS \"Details\", a.name AS \"Name\" FROM commits AS c JOIN _accounts AS a ON c.author_id = a.account_id WHERE $__timeFilter(authored_date) UNION SELECT TO_CHAR(CAST(created_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", created_date AS \"Time\", 'Open a PR' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT TO_CHAR(CAST(merged_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", merged_date AS \"Time\", 'PR gets merged' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(merged_date) UNION SELECT TO_CHAR(CAST(prc.created_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", prc.created_date AS \"Time\", 'Comment on PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type = 'NORMAL' AND $__timeFilter(prc.created_date) UNION SELECT TO_CHAR(CAST(prc.created_date AS TIMESTAMP), 'YYYY-MM-DD | FMDay') AS \"Date\", prc.created_date AS \"Time\", 'Review PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type IN ('REVIEW', 'DIFF') AND $__timeFilter(prc.created_date)) AS t ORDER BY \"Time\" DESC NULLS LAST) SELECT CASE WHEN _row_number = 1 THEN 'Date' ELSE NULL END AS \"Date\", 'Time', \"Activity\", \"Details\", \"Name\" FROM _activities", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Activities by date", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byType", + "options": "number" + }, + "properties": [ + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "#ffffff", + "value": null + }, + { + "color": "super-light-green", + "value": 1 + }, + { + "color": "green", + "value": 5 + }, + { + "color": "dark-green", + "value": 20 + } + ] + } + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + } + ] + }, + { + "matcher": { + "id": "byType", + "options": "number" + }, + "properties": [ + { + "id": "custom.width", + "value": 70 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "week_name" + }, + "properties": [ + { + "id": "custom.width", + "value": 208 + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 10, + "x": 0, + "y": 21 + }, + "id": 23, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _activities AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY \"Date\" ORDER BY \"Time\" DESC NULLS LAST) AS _row_number FROM (SELECT CAST(created_date AS DATE) AS \"Date\", created_date AS \"Time\", 'Create an issue' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.creator_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT CAST(resolution_date AS DATE) AS \"Date\", resolution_date AS \"Time\", 'Issue resolved' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.assignee_id = a.account_id WHERE $__timeFilter(resolution_date) UNION SELECT CAST(authored_date AS DATE) AS \"Date\", authored_date AS \"Time\", 'Finish a commit' AS \"Activity\", message || ' #' || sha AS \"Details\", a.name AS \"Name\" FROM commits AS c JOIN _accounts AS a ON c.author_id = a.account_id WHERE $__timeFilter(authored_date) UNION SELECT CAST(created_date AS DATE) AS \"Date\", created_date AS \"Time\", 'Open a PR' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT CAST(merged_date AS DATE) AS \"Date\", merged_date AS \"Time\", 'PR gets merged' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(merged_date) UNION SELECT CAST(prc.created_date AS DATE) AS \"Date\", prc.created_date AS \"Time\", 'Comment on PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type = 'NORMAL' AND $__timeFilter(prc.created_date) UNION SELECT CAST(prc.created_date AS DATE) AS \"Date\", prc.created_date AS \"Time\", 'Review PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type IN ('REVIEW', 'DIFF') AND $__timeFilter(prc.created_date)) AS t ORDER BY \"Time\" DESC NULLS LAST), _activity_count_per_day AS (SELECT \"Date\", COUNT(*) AS value FROM _activities GROUP BY 1), last_few_calendar_months AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS d, TO_CHAR(CAST(CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS TIMESTAMP), 'wIW YYYY') AS week_name, TO_CHAR(CAST(CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS TIMESTAMP), 'YYYYIW') AS week_number FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _calendar_months_with_rank AS (SELECT d, week_name || ' (' || TO_CHAR(CAST(d - (EXTRACT(ISODOW FROM d) - 1) * INTERVAL '1 day' AS TIMESTAMP), 'MM/DD') || ' ~ ' || TO_CHAR(CAST(d - (EXTRACT(ISODOW FROM d) - 1) * INTERVAL '1 day' + INTERVAL '6 DAY' AS TIMESTAMP), 'MM/DD') || ')' AS week_name, \"week_number\", TO_CHAR(CAST(d AS TIMESTAMP), 'FMDay') AS weekday, DENSE_RANK() OVER (ORDER BY week_number DESC NULLS LAST) AS week_rank FROM last_few_calendar_months ORDER BY 1 DESC NULLS LAST), _final_dataset AS (SELECT _calendar_months_with_rank.*, CASE WHEN _activity_count_per_day.value IS NULL THEN 0 ELSE _activity_count_per_day.value END AS activity_count FROM _calendar_months_with_rank LEFT JOIN _activity_count_per_day ON _calendar_months_with_rank.d = _activity_count_per_day.\"Date\"), WeekSummary AS (SELECT week_name, SUM(CASE WHEN weekday = 'Monday' THEN activity_count END) AS \"Mon\", SUM(CASE WHEN weekday = 'Tuesday' THEN activity_count END) AS \"Tue\", SUM(CASE WHEN weekday = 'Wednesday' THEN activity_count END) AS \"Wed\", SUM(CASE WHEN weekday = 'Thursday' THEN activity_count END) AS \"Thur\", SUM(CASE WHEN weekday = 'Friday' THEN activity_count END) AS \"Fri\", SUM(CASE WHEN weekday = 'Saturday' THEN activity_count END) AS \"Sat\", SUM(CASE WHEN weekday = 'Sunday' THEN activity_count END) AS \"Sun\" FROM _final_dataset WHERE week_rank BETWEEN 1 AND 52 GROUP BY week_name) SELECT week_name, \"Mon\", \"Tue\", \"Wed\", \"Thur\", \"Fri\", \"Sat\", \"Sun\" FROM WeekSummary ORDER BY CAST(SUBSTRING(week_name FROM POSITION(' ' IN week_name) + 1 FOR 4) AS BIGINT) DESC NULLS LAST, CAST(SUBSTRING(week_name FROM 2 FOR POSITION(' ' IN week_name) - 2) AS BIGINT) DESC NULLS LAST", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Activity count in the last year", + "type": "table" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "value" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Create an issue" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Issue resolved" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Finish a commit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Open a PR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PR gets merged" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Comment on PR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Review PR" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 14, + "x": 10, + "y": 21 + }, + "id": 22, + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "normal", + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _activities AS (SELECT *, ROW_NUMBER() OVER (PARTITION BY \"Date\" ORDER BY \"Time\" DESC NULLS LAST) AS _row_number FROM (SELECT CAST(created_date AS DATE) AS \"Date\", created_date AS \"Time\", 'Create an issue' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.creator_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT CAST(resolution_date AS DATE) AS \"Date\", resolution_date AS \"Time\", 'Issue resolved' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.assignee_id = a.account_id WHERE $__timeFilter(resolution_date) UNION SELECT CAST(authored_date AS DATE) AS \"Date\", authored_date AS \"Time\", 'Finish a commit' AS \"Activity\", message || ' #' || sha AS \"Details\", a.name AS \"Name\" FROM commits AS c JOIN _accounts AS a ON c.author_id = a.account_id WHERE $__timeFilter(authored_date) UNION SELECT CAST(created_date AS DATE) AS \"Date\", created_date AS \"Time\", 'Open a PR' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(created_date) UNION SELECT CAST(merged_date AS DATE) AS \"Date\", merged_date AS \"Time\", 'PR gets merged' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(merged_date) UNION SELECT CAST(prc.created_date AS DATE) AS \"Date\", prc.created_date AS \"Time\", 'Comment on PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type = 'NORMAL' AND $__timeFilter(prc.created_date) UNION SELECT CAST(prc.created_date AS DATE) AS \"Date\", prc.created_date AS \"Time\", 'Review PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\" FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE prc.type IN ('REVIEW', 'DIFF') AND $__timeFilter(prc.created_date)) AS t ORDER BY \"Time\" DESC NULLS LAST), _activities_per_day AS (SELECT \"Date\", SUM(CASE WHEN \"Activity\" = 'Create an issue' THEN 1 ELSE 0 END) AS \"Create an issue\", SUM(CASE WHEN \"Activity\" = 'Issue resolved' THEN 1 ELSE 0 END) AS \"Issue resolved\", SUM(CASE WHEN \"Activity\" = 'Finish a commit' THEN 1 ELSE 0 END) AS \"Finish a commit\", SUM(CASE WHEN \"Activity\" = 'Open a PR' THEN 1 ELSE 0 END) AS \"Open a PR\", SUM(CASE WHEN \"Activity\" = 'PR gets merged' THEN 1 ELSE 0 END) AS \"PR gets merged\", SUM(CASE WHEN \"Activity\" = 'Comment on PR' THEN 1 ELSE 0 END) AS \"Comment on PR\", SUM(CASE WHEN \"Activity\" = 'Review PR' THEN 1 ELSE 0 END) AS \"Review PR\" FROM _activities GROUP BY 1), _calendar_dates AS (/* construct the last few calendar months within the selected time period in the top-right corner */ SELECT CAST(($__timeTo()::timestamp - INTERVAL '1 day') AS DATE) AS \"Date\" FROM (SELECT 0 AS \"H\" UNION ALL SELECT 100 UNION ALL SELECT 200 UNION ALL SELECT 300) AS \"H\" CROSS JOIN (SELECT 0 AS \"T\" UNION ALL SELECT 10 UNION ALL SELECT 20 UNION ALL SELECT 30 UNION ALL SELECT 40 UNION ALL SELECT 50 UNION ALL SELECT 60 UNION ALL SELECT 70 UNION ALL SELECT 80 UNION ALL SELECT 90) AS \"T\" CROSS JOIN (SELECT 0 AS \"U\" UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9) AS \"U\" WHERE ($__timeTo()::timestamp - INTERVAL '1 day') > $__timeFrom()), _last_7_days AS (SELECT * FROM _calendar_dates ORDER BY \"Date\" DESC NULLS LAST LIMIT 7) SELECT TO_CHAR(CAST(_last_7_days.\"Date\" AS TIMESTAMP), 'DD/MM %a') AS d, _activities_per_day.* FROM _last_7_days LEFT JOIN _activities_per_day ON _last_7_days.\"Date\" = _activities_per_day.\"Date\" ORDER BY _last_7_days.\"Date\" ASC NULLS FIRST", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Activity distribution in last 7 days", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 10, + "panels": [], + "title": "Throughput", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "TODO" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#f2f1eb", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "DONE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IN_PROGRESS" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 33 + }, + "id": 4, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _issues AS (SELECT TO_CHAR(CAST(created_date AS TIMESTAMP), 'DD/MM/YYYY') AS \"Date\", created_date AS \"Time\", 'Create an issue' AS \"Activity\", '#' || issue_key || ' ' || title AS \"Details\", status, a.name AS \"Name\" FROM issues AS i JOIN _accounts AS a ON i.creator_id = a.account_id WHERE $__timeFilter(created_date)) SELECT status, COUNT(*) FROM _issues GROUP BY 1", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Statuses of assigned issues", + "type": "piechart" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 4, + "y": 33 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name, a.email FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))) SELECT COUNT(DISTINCT c.sha) FROM commits AS c JOIN _accounts AS a ON c.author_id = a.email WHERE $__timeFilter(authored_date)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Code commits", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 9, + "y": 33 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _commits AS (SELECT DISTINCT TO_CHAR(CAST(authored_date AS TIMESTAMP), 'DD/MM/YYYY') AS \"Date\", authored_date AS \"Time\", 'Finish a commit' AS \"Activity\", message || ' #' || sha AS \"Details\", a.name AS \"Name\", c.additions, c.deletions FROM commits AS c JOIN _accounts AS a ON c.author_id = a.account_id WHERE $__timeFilter(authored_date)) SELECT SUM(additions + deletions) FROM _commits", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Lines of code", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 33 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _prs AS (SELECT pr.id AS pr_id, pr.merged_date, '#' || pr.pull_request_key || ' ' || pr.title AS details, prc.id AS comment_id, prc.created_date AS comment_date, a.name, prc.type AS comment_type FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id LEFT JOIN _accounts AS a ON prc.account_id = a.account_id WHERE $__timeFilter(pr.created_date)) SELECT COUNT(DISTINCT CASE WHEN NOT name IS NULL THEN pr_id END) || '/' || COUNT(DISTINCT pr_id) AS text FROM _prs", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "The count of PRs reviewed or commented by the selected users / Total PRs created within this timeframe.", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 0, + "y": 41 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "/.*/", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _prs AS (SELECT DISTINCT TO_CHAR(CAST(created_date AS TIMESTAMP), 'DD/MM/YYYY') AS \"Date\", created_date AS \"Time\", 'Open a PR' AS \"Activity\", '#' || pull_request_key || ' ' || title AS \"Details\", a.name AS \"Name\", pr.id, pr.merged_date FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(created_date)) SELECT COUNT(CASE WHEN NOT merged_date IS NULL THEN id ELSE NULL END) || '/' || COUNT(DISTINCT id) FROM _prs", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "PRs merged / PRs created by the selected users", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 15, + "x": 9, + "y": 41 + }, + "id": 18, + "links": [ + { + "targetBlank": true, + "title": "PR Count", + "url": "https://devlake.apache.org/docs/Metrics/PRCount" + } + ], + "options": { + "barRadius": 0, + "barWidth": 0.5, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "text": { + "valueSize": 12 + }, + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "group": [], + "hide": false, + "metricColumn": "none", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _prs AS (SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COUNT(DISTINCT pr.id) AS pr_count FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(created_date) AND /* and created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -EXTRACT(DAY FROM $__timeFrom())+1 DAY), INTERVAL +1 MONTH) */ NOT pr.merged_date IS NULL GROUP BY 1) SELECT TO_CHAR(CAST(time AS TIMESTAMP), 'FMMonth YYYY') AS month, pr_count AS \"Pull Request Count\" FROM _prs ORDER BY time NULLS FIRST", + "refId": "A", + "select": [ + [ + { + "params": [ + "id" + ], + "type": "column" + } + ] + ], + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "ae_projects", + "timeColumn": "ae_create_time", + "timeColumnType": "timestamp", + "where": [ + { + "name": "$__timeFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "The monthly count of PRs merged out of the total created by the selected users", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 49 + }, + "id": 11, + "panels": [], + "title": "Batch Size and Latency", + "type": "row" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "lines of code" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 50 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _commits AS (SELECT DISTINCT TO_CHAR(CAST(authored_date AS TIMESTAMP), 'DD/MM/YYYY') AS \"Date\", authored_date AS \"Time\", 'Finish a commit' AS \"Activity\", message || ' #' || sha AS \"Details\", a.name AS \"Name\", c.additions, c.deletions, c.sha FROM commits AS c JOIN _accounts AS a ON c.author_id = a.account_id WHERE $__timeFilter(authored_date)), _pr_commits_data AS (SELECT pr.id AS pr_id, pr.merge_commit_sha, SUM(c.additions) + SUM(c.deletions) AS loc FROM pull_requests AS pr LEFT JOIN _commits AS c ON pr.merge_commit_sha = c.sha WHERE $__timeFilter(pr.created_date) AND pr.status = 'MERGED' GROUP BY 1, 2) SELECT AVG(loc) AS \"PR Merged Size\" FROM _pr_commits_data", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Average PR merged size", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "comments per PR" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 5, + "y": 50 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))), _prs AS (SELECT TO_CHAR(CAST(prc.created_date AS TIMESTAMP), 'DD/MM/YYYY') AS \"Date\", prc.created_date AS \"Time\", 'Comment on PR' AS \"Activity\", '#' || pr.pull_request_key || ' ' || pr.title AS \"Details\", a.name AS \"Name\", pr.id AS pr_id, prc.id AS prc_id FROM pull_request_comments AS prc LEFT JOIN pull_requests AS pr ON prc.pull_request_id = pr.id JOIN _accounts AS a ON prc.account_id = a.account_id WHERE $__timeFilter(prc.created_date)) SELECT ROUND(CAST(CAST(COUNT(DISTINCT prc_id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr_id), 0) AS DECIMAL), 1) FROM _prs", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Average review depth", + "type": "stat" + }, + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "description": "average pr review time from first comment to the last comment", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "m" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 10, + "y": 50 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "13.0.2", + "targets": [ + { + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH _accounts AS (SELECT ua.account_id, ua.user_id, u.name FROM accounts AS a JOIN user_accounts AS ua ON a.id = ua.account_id JOIN users AS u ON ua.user_id = u.id WHERE ('${users:csv}' = '' OR ua.user_id::text = ANY(ARRAY[${users:singlequote}]::text[]))) SELECT AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(60, 0)) AS \"Time to merge\" FROM pull_requests AS pr JOIN _accounts AS a ON pr.author_id = a.account_id WHERE $__timeFilter(pr.created_date)", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Average PR time to merge", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "Developer", + "Highlights", + "New" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "None", + "value": "" + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT name || '--' || id FROM teams", + "hide": 0, + "includeAll": false, + "label": "Team", + "multi": false, + "name": "team", + "options": [], + "query": "SELECT name || '--' || id FROM teams", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": {"type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api"}, + "definition": "SELECT users.name || '--' || users.id FROM users LEFT JOIN team_users ON users.id = team_users.user_id WHERE ('${team:csv}' = '' OR team_users.team_id::text = ANY(ARRAY[${team:singlequote}]::text[]))", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "users", + "options": [], + "query": "SELECT users.name || '--' || users.id FROM users LEFT JOIN team_users ON users.id = team_users.user_id WHERE ('${team:csv}' = '' OR team_users.team_id::text = ANY(ARRAY[${team:singlequote}]::text[]))", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "utc", + "title": "Work Logs", + "uid": "d449042e-22f0-4357-b8b7-22083f47618d-pg", + "version": 3, + "weekStart": "" +} \ No newline at end of file diff --git a/grafana/provisioning/dashboards/dashboard.yml b/grafana/provisioning/dashboards/dashboard.yml index 3e10a42cba7..72df91b7701 100644 --- a/grafana/provisioning/dashboards/dashboard.yml +++ b/grafana/provisioning/dashboards/dashboard.yml @@ -38,4 +38,4 @@ providers: # path to dashboard files on disk. Required path: /etc/grafana/dashboards # use folder names from filesystem to create folders in Grafana - foldersFromFilesStructure: true + foldersFromFilesStructure: false diff --git a/grafana/scripts/convert-mysql-to-postgresql.py b/grafana/scripts/convert-mysql-to-postgresql.py new file mode 100755 index 00000000000..b4777b84114 --- /dev/null +++ b/grafana/scripts/convert-mysql-to-postgresql.py @@ -0,0 +1,1409 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Convert Grafana dashboards from MySQL to PostgreSQL datasource. +Uses sqlglot for AST-based SQL parsing and transformation. +""" + +import json +import sys +import re +from pathlib import Path +from typing import Any + +try: + import sqlglot +except ImportError: + print("ERROR: sqlglot not installed. Run: pip install -r requirements.txt") + sys.exit(1) + + +def extract_balanced_parens(text: str, start_pos: int) -> tuple[str, int] | None: + """ + Extract content from start_pos to matching closing paren with balanced counting. + Returns (extracted_content, closing_paren_position) or None if no match. + """ + paren_depth = 0 + i = start_pos + + while i < len(text): + if text[i] == '(': + paren_depth += 1 + elif text[i] == ')': + if paren_depth == 0: + return (text[start_pos:i], i) + paren_depth -= 1 + i += 1 + + return None + + +def convert_sql_mysql_to_postgres(sql: str) -> str: + """ + Convert MySQL SQL to PostgreSQL using sqlglot AST parsing. + Falls back to regex for patterns sqlglot doesn't handle. + """ + if not sql or not sql.strip(): + return sql + + try: + # Protect string literals from function conversions + sql, string_literals = protect_string_literals(sql) + + # Protect Grafana template variables from sqlglot conversion + # sqlglot treats ${var} as MySQL user variables → STRUCT(var) + sql, placeholders = protect_grafana_variables(sql) + + # Try sqlglot transpilation first + converted = sqlglot.transpile(sql, read='mysql', write='postgres')[0] + + # Post-process patterns sqlglot might miss + converted = post_process_sql(converted) + + # Restore Grafana macros to lowercase (sqlglot uppercases them) + converted = restore_grafana_macros(converted) + + # Fix $__timeFilter(MAX/MIN(...)) macro expansion issue (Step 5) + # Must run AFTER restore_grafana_macros() to match lowercase $__timeFilter + converted = fix_timefilter_aggregate_macro(converted) + + # Restore Grafana template variables + converted = restore_grafana_variables(converted, placeholders) + + # Convert $interval(expr) pattern - Grafana variable used as function name + # When $interval = 'DAYOFMONTH', becomes DAYOFMONTH(expr) + # When $interval = 'WEEKDAY', becomes WEEKDAY(expr) + # Replace with CASE statement for PostgreSQL + pattern = re.compile(r'\$interval\s*\(', re.IGNORECASE) + result_interval = [] + pos_interval = 0 + + for match in pattern.finditer(converted): + result_interval.append(converted[pos_interval:match.start()]) + extracted = extract_balanced_parens(converted, match.end()) + if extracted: + expr, closing_pos = extracted + replacement = f"CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM {expr}) ELSE (EXTRACT(ISODOW FROM {expr}) - 1) END" + result_interval.append(replacement) + pos_interval = closing_pos + 1 + else: + result_interval.append(match.group(0)) + pos_interval = match.end() + + result_interval.append(converted[pos_interval:]) + converted = ''.join(result_interval) + + # Restore string literals + converted = restore_string_literals(converted, string_literals) + + # Convert MySQL date format patterns to PostgreSQL (Step 1 - after string restoration) + converted = convert_mysql_date_formats(converted) + + # Convert INTERVAL 'n DAY' * WEEKDAY(expr) -> (EXTRACT(ISODOW FROM expr) - 1) * INTERVAL 'n day' + # Also handle negative case: INTERVAL 'n DAY' * -WEEKDAY(expr) -> -(EXTRACT(ISODOW FROM expr) - 1) * INTERVAL 'n day' + # This pattern appears after sqlglot converts DATE_SUB(x, INTERVAL WEEKDAY(y) DAY) + # Manual string building to handle nested parentheses properly + pattern = re.compile(r"INTERVAL\s+'(\d+)\s+DAY'\s*\*\s*(-?)\s*WEEKDAY\s*\(", re.IGNORECASE) + result = [] + pos = 0 + + for match in pattern.finditer(converted): + interval_num = match.group(1) + negative_sign = match.group(2) + result.append(converted[pos:match.start()]) + + extracted = extract_balanced_parens(converted, match.end()) + if extracted: + expr, closing_pos = extracted + if negative_sign: + replacement = f"{negative_sign}(EXTRACT(ISODOW FROM {expr}) - 1) * INTERVAL '{interval_num} day'" + else: + replacement = f"(EXTRACT(ISODOW FROM {expr}) - 1) * INTERVAL '{interval_num} day'" + result.append(replacement) + pos = closing_pos + 1 + else: + result.append(match.group(0)) + pos = match.end() + + # Add remaining text after last match + result.append(converted[pos:]) + converted = ''.join(result) + + # Convert standalone WEEKDAY(expr) calls (not multiplication pattern) + # WEEKDAY(date) returns 0-6 (Mon-Sun) in MySQL, ISODOW returns 1-7 (Mon-Sun) in PostgreSQL + # Pattern: WEEKDAY(...) BETWEEN/=//>=/<= + # Replace with: (EXTRACT(ISODOW FROM ...) - 1) to maintain 0-6 range + pattern2 = re.compile(r"\bWEEKDAY\s*\(", re.IGNORECASE) + result2 = [] + pos2 = 0 + + for match in pattern2.finditer(converted): + result2.append(converted[pos2:match.start()]) + + extracted = extract_balanced_parens(converted, match.end()) + if extracted: + expr, closing_pos = extracted + replacement = f"(EXTRACT(ISODOW FROM {expr}) - 1)" + result2.append(replacement) + pos2 = closing_pos + 1 + else: + result2.append(match.group(0)) + pos2 = match.end() + + # Add remaining text after last match + result2.append(converted[pos2:]) + converted = ''.join(result2) + + # DIV(TO_CHAR(date, 'YYYY-MM'), N) -> calculate month-based division (after format conversion) + # Used for bucketing dates into N-month periods (e.g., half-years with N=6) + converted = re.sub( + r'DIV\s*\(\s*TO_CHAR\s*\(\s*([^,]+?)\s*,\s*["\']YYYY-MM["\']\s*\)\s*,\s*(\d+)\s*\)', + r'((EXTRACT(YEAR FROM \1) * 12 + EXTRACT(MONTH FROM \1)) / \2)', + converted, + flags=re.IGNORECASE + ) + + # Convert AS 'alias' to AS "alias" after string restoration + # PostgreSQL doesn't accept single quotes for column aliases + # Skip SQL type keywords (DATE, TIMESTAMP, etc.) - they should remain unquoted + def convert_single_quoted_alias(match): + alias = match.group(1) + alias_upper = alias.upper() + sql_type_keywords = {'DATE', 'TIME', 'TIMESTAMP', 'INT', 'INTEGER', 'BIGINT', 'SMALLINT', + 'DECIMAL', 'NUMERIC', 'TEXT', 'VARCHAR', 'CHAR', 'BOOLEAN', 'BOOL', + 'REAL', 'DOUBLE', 'FLOAT'} + # Only treat as type keyword if exact uppercase match (e.g., 'TIMESTAMP' not 'Timestamp') + if alias_upper in sql_type_keywords and alias == alias_upper: + return f'AS {alias}' # Unquote type keyword + return f'AS "{alias}"' # Quote regular alias + + converted = re.sub(r'\bAS\s+\'([^\']+)\'', convert_single_quoted_alias, converted, flags=re.IGNORECASE) + + # Quote bare (unquoted) column aliases to ensure case-sensitive matching + # sqlglot inconsistently quotes aliases: subquery has "AS Date", outer query has "\"Date\"" + # PostgreSQL folds unquoted identifiers to lowercase, causing column not found errors + # Pattern: AS (not already quoted, not a SQL type) + # Exclude SQL types: DATE, TIME, TIMESTAMP, INT, INTEGER, BIGINT, TEXT, VARCHAR, etc. + sql_types = r'(?!(?:DATE|TIME|TIMESTAMP|INT|INTEGER|BIGINT|SMALLINT|DECIMAL|NUMERIC|TEXT|VARCHAR|CHAR|BOOLEAN|BOOL|REAL|DOUBLE|FLOAT)\b)' + converted = re.sub( + rf'(? "value", != "value", IN ("val1", "val2"), CONCAT(..., "text", ...), THEN "value" + # But skip: column aliases after AS (already handled above as AS "alias") + # MySQL accepts both quotes for strings, PostgreSQL only single quotes + converted = re.sub(r'(=|<>|!=)\s+"([^"]+)"', r"\1 '\2'", converted) + converted = re.sub(r'\bIN\s*\(\s*"([^"]+)"\s*\)', r"IN ('\1')", converted, flags=re.IGNORECASE) + + # Convert CASE THEN/ELSE double-quoted strings to single quotes + # THEN "text" → THEN 'text', ELSE "text" → ELSE 'text' + converted = re.sub(r'\bTHEN\s+"([^"]+)"', r"THEN '\1'", converted, flags=re.IGNORECASE) + converted = re.sub(r'\bELSE\s+"([^"]+)"', r"ELSE '\1'", converted, flags=re.IGNORECASE) + + # Convert function argument double-quoted strings to single quotes + # Common in SPLIT_PART, CONCAT, etc.: func(..., "delimiter", ...) → func(..., 'delimiter', ...) + # Match: comma followed by whitespace, double-quoted string, optional whitespace, comma or closing paren + converted = re.sub(r',(\s*)"([^"]+)"(\s*[,)])', r",\1'\2'\3", converted) + + # Convert MySQL FIELD() function to PostgreSQL CASE WHEN + # MySQL: ORDER BY FIELD(column, 'val1', 'val2', 'val3') + # PostgreSQL: ORDER BY CASE column WHEN 'val1' THEN 1 WHEN 'val2' THEN 2 WHEN 'val3' THEN 3 END + def convert_field_function(match): + column = match.group(1).strip() + values_str = match.group(2).strip() + # Split values by comma, handle quoted strings + values = [v.strip() for v in re.findall(r"'[^']*'|[^,]+", values_str)] + + when_clauses = [] + for i, value in enumerate(values, start=1): + when_clauses.append(f"WHEN {value} THEN {i}") + + return f"CASE {column} {' '.join(when_clauses)} END" + + converted = re.sub( + r'\bFIELD\s*\(\s*([^,]+)\s*,\s*([^)]+)\s*\)', + convert_field_function, + converted, + flags=re.IGNORECASE + ) + + # Convert double-quoted literals inside CONCAT to single quotes + # CONCAT(..., "(text)", ...) → CONCAT(..., '(text)', ...) + def replace_concat_strings(match): + content = match.group(0) + # Replace double-quoted strings inside CONCAT with single quotes + content = re.sub(r'"([^"]+)"', r"'\1'", content) + return content + + converted = re.sub(r'\bCONCAT\s*\([^)]+\)', replace_concat_strings, converted, flags=re.IGNORECASE) + + # Convert THEN/ELSE double-quoted strings to single quotes + # THEN "text" → THEN 'text', ELSE "text" → ELSE 'text' + converted = re.sub(r'\b(THEN|ELSE)\s+"([^"]+)"', r"\1 '\2'", converted, flags=re.IGNORECASE) + + # Convert SELECT "string literal" AS to SELECT 'string literal' AS + # Only convert if string contains spaces (indicates literal not identifier) + converted = re.sub(r'\bSELECT\s+"([^"]*\s[^"]*)"(\s+AS\s+)', r"SELECT '\1'\2", converted, flags=re.IGNORECASE) + + # Convert concatenation "string literal" to 'string literal' + # Pattern: || "text" becomes || 'text' + converted = re.sub(r'\|\|\s*"([^"]+)"', r"|| '\1'", converted) + + # Fix EXISTS(SELECT COUNT(...)) = 0/1 patterns that cause boolean = integer error + # MySQL: EXISTS(SELECT COUNT(...)) = 0 returns true/false, PostgreSQL: EXISTS returns boolean, can't compare to int + # Pattern 1: EXISTS(SELECT COUNT(...) FROM ...) = 0 → NOT EXISTS(SELECT 1 FROM ...) + converted = re.sub( + r'EXISTS\s*\(\s*SELECT\s+COUNT\s*\([^)]*\)\s+FROM\s+([^)]+)\)\s*=\s*0', + r'NOT EXISTS(SELECT 1 FROM \1)', + converted, + flags=re.IGNORECASE + ) + # Pattern 2: EXISTS(SELECT COUNT(...) FROM ...) = 1 → EXISTS(SELECT 1 FROM ...) + converted = re.sub( + r'EXISTS\s*\(\s*SELECT\s+COUNT\s*\([^)]*\)\s+FROM\s+([^)]+)\)\s*=\s*1', + r'EXISTS(SELECT 1 FROM \1)', + converted, + flags=re.IGNORECASE + ) + + # Ambiguous alias fix DISABLED - causes incorrect _computed suffix duplication + # The pattern was too greedy and matched cases where aliases are reused across CTEs + # Example issue: adoption_pct → adoption_pct_computed → adoption_pct_computed_computed + # If specific ambiguous column errors appear, handle them case-by-case + + # Fix mixed-type CASE expressions (TEXT and NUMERIC) + # PostgreSQL: all CASE branches must be same type + # Pattern: CASE with THEN 'string' and THEN numeric/arithmetic + # Solution: Cast arithmetic branches to ::TEXT + # Example: THEN value/60 ELSE 'No data' → THEN (value/60)::TEXT ELSE 'No data' + def cast_numeric_then_clause(match): + # Match captures: group(1) = THEN expression, group(2) = ELSE string + then_expr = match.group(1).strip() + else_str = match.group(2) + # If expression contains division or CAST/NULLIF (likely numeric), wrap in ()::TEXT + if '/' in then_expr or 'CAST' in then_expr.upper() or 'NULLIF' in then_expr.upper(): + # Wrap in parens if not already wrapped + if not (then_expr.startswith('(') and then_expr.endswith(')')): + then_expr = f'({then_expr})' + return f'THEN {then_expr}::TEXT ELSE \'{else_str}\'' + return match.group(0) + + # Match: THEN ELSE 'string_literal' + # Negative lookahead (?!.*\bWHEN\b) ensures we don't match across WHEN boundaries + converted = re.sub( + r'\bTHEN\s+((?!.*\bWHEN\b).+?)\s+ELSE\s+\'([^\']+)\'', + cast_numeric_then_clause, + converted, + flags=re.IGNORECASE | re.DOTALL + ) + + # Convert SUBSTRING_INDEX to PostgreSQL equivalent + # MySQL: SUBSTRING_INDEX(str, delim, count) - get substring before/after delimiter occurrence + # PostgreSQL: SPLIT_PART for positive count, REVERSE+SPLIT_PART for negative count + def convert_substring_index(match): + str_arg = match.group(1).strip() + delim_arg = match.group(2).strip() + count_arg = match.group(3).strip() + + # Check if count is negative (get from end) + if count_arg.startswith('-'): + # Negative count: get from end + # SUBSTRING_INDEX(str, delim, -1) → REVERSE(SPLIT_PART(REVERSE(str), delim, 1)) + positive_count = count_arg[1:] # Remove minus sign + return f'REVERSE(SPLIT_PART(REVERSE({str_arg}), {delim_arg}, {positive_count}))' + else: + # Positive count: get from beginning + # SUBSTRING_INDEX(str, delim, 1) → SPLIT_PART(str, delim, 1) + return f'SPLIT_PART({str_arg}, {delim_arg}, {count_arg})' + + # Match: SUBSTRING_INDEX(str, delim, count) + converted = re.sub( + r'\bSUBSTRING_INDEX\s*\(\s*([^,]+),\s*([^,]+),\s*([^)]+)\)', + convert_substring_index, + converted, + flags=re.IGNORECASE + ) + + # Fix STRING_AGG double-quoted separator (from sqlglot GROUP_CONCAT conversion) + # PostgreSQL: STRING_AGG(expr, "sep") → STRING_AGG(expr, 'sep') + # sqlglot converts GROUP_CONCAT but keeps separator quoting from source + # Use greedy match to handle expressions with internal commas + converted = re.sub( + r'\bSTRING_AGG\s*\((.+),\s*"([^"]*)"\s*\)', + r"STRING_AGG(\1, '\2')", + converted, + flags=re.IGNORECASE + ) + + # Fix HAVING clause with column alias - PostgreSQL doesn't allow aliases in HAVING + # MySQL: HAVING `alias` IS NOT NULL + # PostgreSQL: Remove HAVING on aggregate NULL checks (naturally filtered) + # Pattern: HAVING NOT "alias" IS NULL or HAVING "alias" IS NOT NULL + converted = re.sub( + r'\bHAVING\s+(?:NOT\s+)?"([^"]+)"\s+IS\s+(?:NOT\s+)?NULL', + '', + converted, + flags=re.IGNORECASE + ) + + # Fix ambiguous identifier in GROUP BY when alias matches column name + # PostgreSQL can't distinguish between alias and column with same name + # Keep GROUP BY as-is for now - dashboards using positional (1,2,3) work fine + # Dashboards using aliases may have ambiguity - fix at dashboard level + def quote_group_by_identifiers(sql_text): + def quote_group_by_item(match): + prefix = match.group(1) # "GROUP BY " + items_str = match.group(2) # rest of clause + + # Keep positional numbers (1,2,3) as-is since they work perfectly + # Keep complex expressions (table.col) as-is + # Only quote simple identifiers for case-sensitivity + items = [] + for item in items_str.split(','): + item = item.strip() + if not item or item.isdigit() or '.' in item: + items.append(item) + elif re.match(r'^\w+$', item): + items.append(f'"{item}"') + else: + items.append(item) + + return prefix + ', '.join(items) + + sql_text = re.sub( + r'\b(GROUP\s+BY\s+)([^;]*?)(?=\s*(?:HAVING|ORDER|LIMIT|$))', + quote_group_by_item, + sql_text, + flags=re.IGNORECASE + ) + return sql_text + + converted = quote_group_by_identifiers(converted) + + # Quote capitalized identifiers in ORDER BY (same issue as GROUP BY) + # Pattern: ORDER BY - quote to match case-sensitive alias + def quote_order_by_identifiers(sql_text): + def quote_order_by_item(match): + prefix = match.group(1) # "ORDER BY " + items_str = match.group(2) # rest before DESC/ASC/NULLS/end + + # Split by comma + items = [] + for item in items_str.split(','): + item = item.strip() + # Match: bare capitalized word (not quoted, not numeric) + word_match = re.match(r'^([A-Z]\w*)$', item) + if word_match and not item.isdigit(): + items.append(f'"{item}"') + elif item and not item.startswith('"'): + items.append(item) + else: + items.append(item) + + return prefix + ', '.join(items) + + # Match ORDER BY clause up to DESC/ASC/NULLS or next keyword + sql_text = re.sub( + r'\b(ORDER\s+BY\s+)([^;]+?)(?=\s+(?:DESC|ASC|NULLS|LIMIT|$))', + quote_order_by_item, + sql_text, + flags=re.IGNORECASE + ) + return sql_text + + converted = quote_order_by_identifiers(converted) + + # Quote capitalized bare identifiers in SELECT column lists + # Pattern: SELECT ..., Capitalized, ... (not after AS, not SQL keywords) + # Must match column references that correspond to quoted aliases from subqueries + def quote_select_columns(sql_text): + # Match SELECT clause, extract column list + def quote_columns_in_select(match): + select_keyword = match.group(1) # "SELECT " or "SELECT DISTINCT " etc + columns_clause = match.group(2) # column list + + # Split by comma, quote capitalized bare identifiers + columns = [] + for col in columns_clause.split(','): + col = col.strip() + # Skip if already quoted, contains operators/functions, or is * + if col.startswith('"') or col == '*' or '(' in col or '.' in col or 'AS ' in col.upper(): + columns.append(col) + # Quote capitalized bare word (e.g., Activity, Details, Name) + elif re.match(r'^[A-Z]\w*$', col): + columns.append(f'"{col}"') + else: + columns.append(col) + + return select_keyword + ', '.join(columns) + + # Match SELECT column list (from SELECT to FROM) + sql_text = re.sub( + r'\b(SELECT(?:\s+DISTINCT)?\s+)(.*?)(?=\s+FROM\b)', + quote_columns_in_select, + sql_text, + flags=re.IGNORECASE + ) + return sql_text + + converted = quote_select_columns(converted) + + # Quote table.Column references where Column is capitalized + # Pattern: table.Date → table."Date" (prevents case-folding in JOIN/WHERE clauses) + # Match: word.Capitalized (not followed by opening paren for functions) + converted = re.sub(r'\.([A-Z]\w*)\b(?!\s*\()', r'."\1"', converted) + + # Quote bare capitalized identifiers in CASE WHEN clauses + # WHEN Activity = 'value' → WHEN "Activity" = 'value' + # Match WHEN case-insensitive, but identifier must start with uppercase + converted = re.sub( + r'\b(?:WHEN|when)\s+([A-Z]\w*)\s*(=|<>|!=|>|<|>=|<=|IN|LIKE|IS)', + r'WHEN "\1" \2', + converted + ) + + # Quote bare capitalized identifiers in WHERE clauses + # WHERE Date > '2020' → WHERE "Date" > '2020' + # Match WHERE case-insensitive, but identifier must start with uppercase + converted = re.sub( + r'\b(?:WHERE|where)\s+([A-Z]\w*)\s*(=|<>|!=|>|<|>=|<=|IN|LIKE|IS)', + r'WHERE "\1" \2', + converted + ) + + # Fix text column comparisons to integers + # MySQL allows implicit conversion, PostgreSQL doesn't + # issue_key <> 0 → issue_key <> '0' (text column needs text literal) + converted = re.sub( + r'\b([a-z_]+)\s*(=|<>|!=)\s*0\b', + r"\1 \2 '0'", + converted + ) + + # Protect equality comparisons with Grafana variables from empty values + # connection_id = ${var} → ('${var}' = '' OR connection_id::text = '${var}') + # table.connection_id = ${var} → ('${var}' = '' OR table.connection_id::text = '${var}') + # When variable empty, condition becomes TRUE (no filter), when set, applies filter + # Cast both sides to text to handle integer/text columns uniformly + def protect_equality_variable(match): + column = match.group(1) + operator = match.group(2) + var = match.group(3) + return f"('{var}' = '' OR {column}::text {operator} '{var}')" + + # Match: column = ${var} or table.column = ${var} + # Capture optional table prefix with (?:\.\w+)? + converted = re.sub( + r'(\w+(?:\.\w+)?)\s*(=|<>|!=)\s*(\$\{[^}]+\}|\$[a-z_][a-z0-9_]*)', + protect_equality_variable, + converted + ) + + # PostgreSQL rejects IN () when Grafana variables are empty + # Use = ANY(ARRAY[...]) instead of IN (...) because empty array is valid SQL + # When empty: = ANY(ARRAY[]::text[]) is always false + # When multi-value: = ANY(ARRAY['val1','val2']::text[]) filters correctly + def format_grafana_variable(var): + """Extract variable name and return (var_formatted, var_check) for Grafana expansion.""" + if var.startswith('${') and var.endswith('}'): + var_name = var[2:-1] + if ':' not in var_name: + return f'${{{var_name}:singlequote}}', f'${{{var_name}:csv}}' + return var, var + # Bare $var form + var_name = var[1:] + return f'${{{var_name}:singlequote}}', f'${{{var_name}:csv}}' + + def protect_in_clause_with_cast(match): + column = match.group(1) + var = match.group(2) + if '::' not in column: + column = f'{column}::text' + var_formatted, var_check = format_grafana_variable(var) + return f"('{var_check}' = '' OR {column} = ANY(ARRAY[{var_formatted}]::text[]))" + + def protect_function_in_clause(match): + func_expr = match.group(1) + var = match.group(2) + var_formatted, var_check = format_grafana_variable(var) + return f"('{var_check}' = '' OR {func_expr} = ANY(ARRAY[{var_formatted}]::text[]))" + + converted = re.sub( + r'([\w.]+(?:::\w+)?)\s+IN\s*\(\s*(\$\{[^}]+\}|\$(?!__)[a-z_][a-z0-9_]*)\s*\)', + protect_in_clause_with_cast, + converted, + flags=re.IGNORECASE + ) + + converted = re.sub( + r'(SPLIT_PART\([^)]+\))\s+IN\s*\(\s*(\$\{[^}]+\}|\$(?!__)[a-z_][a-z0-9_]*)\s*\)', + protect_function_in_clause, + converted, + flags=re.IGNORECASE + ) + + return converted + except Exception as e: + # Fallback to regex-based conversion for complex cases + print(f" [WARN] sqlglot failed, using regex fallback: {str(e)[:100]}") + fallback = regex_fallback_conversion(sql) + fallback = restore_grafana_variables(fallback, placeholders) + fallback = restore_string_literals(fallback, string_literals) + return fallback + + +def protect_string_literals(sql_text: str) -> tuple[str, dict]: + """ + Replace all string literals with placeholders to protect them from conversion. + Returns modified SQL and mapping of placeholders to original strings. + Handles SQL comments (-- and /* */) to avoid treating quotes in comments as string delimiters. + """ + literals = {} + counter = 0 + result = [] + i = 0 + + while i < len(sql_text): + # Skip single-line comments (-- comment) + if i < len(sql_text) - 1 and sql_text[i:i+2] == '--': + # Find end of line + end = sql_text.find('\n', i) + if end == -1: + result.append(sql_text[i:]) + break + else: + result.append(sql_text[i:end+1]) + i = end + 1 + continue + + # Skip multi-line comments (/* comment */) + if i < len(sql_text) - 1 and sql_text[i:i+2] == '/*': + end = sql_text.find('*/', i + 2) + if end == -1: + result.append(sql_text[i:]) + break + else: + result.append(sql_text[i:end+2]) + i = end + 2 + continue + + # Process string literals + if sql_text[i] in ("'", '"'): + quote_char = sql_text[i] + literal_parts = [sql_text[i]] + i += 1 + while i < len(sql_text): + literal_parts.append(sql_text[i]) + if sql_text[i] == quote_char: + # Check for escaped quote ('' or "") + if i + 1 < len(sql_text) and sql_text[i + 1] == quote_char: + literal_parts.append(sql_text[i + 1]) + i += 2 + else: + i += 1 + break + else: + i += 1 + + literal = ''.join(literal_parts) + placeholder = f'STRING_LITERAL_{counter}_PLACEHOLDER' + literals[placeholder] = literal + result.append(placeholder) + counter += 1 + else: + result.append(sql_text[i]) + i += 1 + + return ''.join(result), literals + + +def restore_string_literals(sql_text: str, literals: dict) -> str: + """Restore string literals from placeholders.""" + for placeholder, literal in literals.items(): + sql_text = sql_text.replace(placeholder, literal) + return sql_text + + +def convert_mysql_date_formats(sql: str) -> str: + """ + Convert MySQL date format specifiers to PostgreSQL in TO_CHAR/TO_DATE calls. + Operates character-by-character within format strings. + Step 1 of the conversion fixes. + """ + # MySQL -> PostgreSQL format mapping + format_map = { + '%Y': 'YYYY', # 4-digit year + '%y': 'YY', # 2-digit year + '%X': 'IYYY', # ISO year + '%V': 'IW', # ISO week + '%m': 'MM', # month number + '%M': 'FMMonth', # full month name + '%d': 'DD', # day of month + '%W': 'FMDay', # full weekday name + '%w': 'D', # day of week number + '%u': 'IW', # ISO week + '%H': 'HH24', # 24-hour + '%i': 'MI', # minutes + '%s': 'SS' # seconds + } + + def convert_format_string(format_str): + """Convert a single format string from MySQL to PostgreSQL.""" + # Skip if already PostgreSQL format (contains YYYY, MM, DD without %) + if not '%' in format_str and any(pg in format_str for pg in ['YYYY', 'MM', 'DD', 'HH24']): + return format_str + + result = [] + i = 0 + while i < len(format_str): + if format_str[i] == '%' and i + 1 < len(format_str): + # Try to match %X format specifier + two_char = format_str[i:i+2] + if two_char in format_map: + result.append(format_map[two_char]) + i += 2 + else: + # Unknown specifier, keep as-is + result.append(format_str[i]) + i += 1 + else: + result.append(format_str[i]) + i += 1 + return ''.join(result) + + # Find and convert TO_CHAR(..., 'format') and TO_DATE(..., 'format') + def replacer(match): + func_name = match.group(1) + first_arg = match.group(2) + format_str = match.group(3) + converted_format = convert_format_string(format_str) + return f"{func_name}({first_arg}, '{converted_format}')" + + # Pattern matches TO_CHAR(expr, 'format') or TO_DATE(expr, 'format') + # Use non-greedy matching for first argument + sql = re.sub( + r'\b(TO_CHAR|TO_DATE)\s*\(\s*([^,]+?)\s*,\s*\'([^\']+)\'\s*\)', + replacer, + sql, + flags=re.IGNORECASE + ) + + return sql + + +def fix_timefilter_aggregate_macro(sql: str) -> str: + """ + Fix $__timeFilter(MAX/MIN(...)) macro expansion that creates MAX(boolean). + Converts: $__timeFilter(MAX(col)) -> MAX(col) BETWEEN $__timeFrom() AND $__timeTo() + Step 5 of the conversion fixes. + Must run AFTER restore_grafana_macros() to match lowercase $__timeFilter. + """ + # Simple pattern for $__timeFilter(MAX(col)) or $__timeFilter(MIN(col)) + # Handles simple column references + sql = re.sub( + r'\$__timeFilter\s*\(\s*(MAX|MIN)\s*\(([^)]+)\)\s*\)', + r'\1(\2) BETWEEN $__timeFrom() AND $__timeTo()', + sql, + flags=re.IGNORECASE + ) + + # For nested expressions like $__timeFilter(MAX(CAST(col AS DATE))), + # use balanced paren matching + pattern = r'\$__timeFilter\s*\(' + result = [] + i = 0 + while i < len(sql): + match = re.match(pattern, sql[i:], re.IGNORECASE) + if match: + start = i + match.end() + # Check if next token is MAX or MIN + agg_match = re.match(r'\s*(MAX|MIN)\s*\(', sql[start:], re.IGNORECASE) + if agg_match: + agg_func = agg_match.group(1).upper() + agg_start = start + agg_match.end() + # Find matching closing paren for MAX/MIN + depth = 1 + j = agg_start + while j < len(sql) and depth > 0: + if sql[j] == '(': + depth += 1 + elif sql[j] == ')': + depth -= 1 + j += 1 + if depth == 0: + # Found closing paren for MAX/MIN + agg_arg = sql[agg_start:j-1] + # Now find closing paren for $__timeFilter + if j < len(sql) and sql[j] == ')': + # Replace with aggregate BETWEEN pattern + result.append(f"{agg_func}({agg_arg}) BETWEEN $__timeFrom() AND $__timeTo()") + i = j + 1 + continue + # Not a MAX/MIN pattern, keep original + result.append(sql[i]) + i += 1 + else: + result.append(sql[i]) + i += 1 + + return ''.join(result) + + +def replace_function_with_balanced_parens(sql_text: str, func_pattern: str, replacement_func) -> str: + """ + Generic function replacer that handles nested parentheses via depth counting. + + Args: + sql_text: SQL string to process (string literals already protected) + func_pattern: Regex pattern for function name (e.g., r'\\bHOUR\\s*\\(') + replacement_func: Function taking matched argument and returning replacement string + + Returns: + Modified SQL with function replaced + """ + result = [] + i = 0 + pattern = re.compile(func_pattern, re.IGNORECASE) + while i < len(sql_text): + match = pattern.search(sql_text, i) + if not match: + result.append(sql_text[i:]) + break + result.append(sql_text[i:match.start()]) + start = match.end() + depth = 1 + j = start + while j < len(sql_text) and depth > 0: + if sql_text[j] == '(': + depth += 1 + elif sql_text[j] == ')': + depth -= 1 + j += 1 + if depth == 0: + arg = sql_text[start:j-1] + result.append(replacement_func(arg)) + i = j + else: + result.append(match.group(0)) + i = start + return ''.join(result) + + +def post_process_sql(sql: str) -> str: + """ + Post-process SQL after sqlglot conversion for Grafana-specific patterns. + """ + # Remove backticks (PostgreSQL uses double quotes or no quotes) + sql = sql.replace('`', '"') + + # CURDATE() -> CURRENT_DATE + sql = re.sub(r'\bCURDATE\s*\(\s*\)', 'CURRENT_DATE', sql, flags=re.IGNORECASE) + + # FORMAT(value, decimals) -> ROUND(value, decimals) + # MySQL FORMAT() formats number with commas and decimal places + # PostgreSQL ROUND() works - auto-casts to text in CONCAT context + # Don't add ::text here because FORMAT has nested functions with commas + # and [^,]+ regex can't handle them properly + sql = re.sub( + r'\bFORMAT\s*\([^)]*\)', + lambda m: m.group(0).replace('FORMAT', 'ROUND', 1), + sql, + flags=re.IGNORECASE + ) + + # Cast Grafana time macros to timestamp when used with INTERVAL + # $__timeFrom() + INTERVAL → $__timeFrom()::timestamp + INTERVAL + # $__timeTo() + INTERVAL → $__timeTo()::timestamp + INTERVAL + # Also handle when wrapped in parentheses: ($__timeTo() - INTERVAL + sql = re.sub( + r'(\$__timeFrom\(\)|\$__timeTo\(\))\s*([+\-])\s*INTERVAL', + r'\1::timestamp \2 INTERVAL', + sql, + flags=re.IGNORECASE + ) + # Handle parentheses-wrapped time macros: ($__timeFrom() or ($__timeTo() + sql = re.sub( + r'\((\$__timeFrom\(\)|\$__timeTo\(\))\s*([+\-])\s*INTERVAL', + r'(\1::timestamp \2 INTERVAL', + sql, + flags=re.IGNORECASE + ) + + # Remove CHARACTER SET from CAST (MySQL-specific) + # CAST(col AS CHAR CHARACTER SET utf8mb4) → CAST(col AS TEXT) + sql = re.sub( + r'\bCAST\s*\(([^)]+)\s+AS\s+CHAR\s+CHARACTER\s+SET\s+\w+\)', + r'CAST(\1 AS TEXT)', + sql, + flags=re.IGNORECASE + ) + + # CAST AS CHAR → CAST AS TEXT (PostgreSQL doesn't have bare CHAR for casting) + # Use global replace to handle nested CAST expressions + sql = re.sub( + r'\s+AS\s+CHAR\)', + ' AS TEXT)', + sql, + flags=re.IGNORECASE + ) + + # HOUR() -> EXTRACT(HOUR FROM ...) + sql = replace_function_with_balanced_parens( + sql, + r'\bHOUR\s*\(', + lambda arg: f'EXTRACT(HOUR FROM {arg})' + ) + + # DAYOFMONTH() -> EXTRACT(DAY FROM ...) (no underscore variant) + sql = replace_function_with_balanced_parens( + sql, + r'\bDAYOFMONTH\s*\(', + lambda arg: f'EXTRACT(DAY FROM {arg})' + ) + + # DAY_OF_MONTH() -> EXTRACT(DAY FROM ...) (with underscore variant) + sql = replace_function_with_balanced_parens( + sql, + r'\bDAY_OF_MONTH\s*\(', + lambda arg: f'EXTRACT(DAY FROM {arg})' + ) + + # DAY() -> EXTRACT(DAY FROM ...) + sql = replace_function_with_balanced_parens( + sql, + r'\bDAY\s*\(', + lambda arg: f'EXTRACT(DAY FROM {arg})' + ) + + # Boolean comparisons: column = 1 → column = TRUE, column = 0 → column = FALSE + # PostgreSQL schema has BOOLEAN type - does NOT accept integer comparison + # ERROR: "operator does not exist: boolean = integer" + # Must convert ALL boolean column comparisons: WHERE/AND/WHEN + # Safe: CASE WHEN bool_col = TRUE THEN 1 - condition is boolean, result is integer + # THEN/ELSE values determine CASE return type, not WHEN condition type + sql = re.sub( + r'(WHERE|AND|WHEN)\s+([\w.]*(?:has_|is_)[\w.]+|[\w.]+_flag)\s*=\s*1\b', + r'\1 \2 = TRUE', + sql, + flags=re.IGNORECASE + ) + sql = re.sub( + r'(WHERE|AND|WHEN)\s+([\w.]*(?:has_|is_)[\w.]+|[\w.]+_flag)\s*=\s*0\b', + r'\1 \2 = FALSE', + sql, + flags=re.IGNORECASE + ) + + # DAY_OF_WEEK -> EXTRACT(ISODOW FROM ...) + # MySQL DAY_OF_WEEK() returns 1=Sunday, 2=Monday, 3=Tuesday...7=Saturday + # PostgreSQL ISODOW returns 1=Monday, 2=Tuesday...7=Sunday + sql = replace_function_with_balanced_parens( + sql, + r'\bDAY_OF_WEEK\s*\(', + lambda arg: f'EXTRACT(ISODOW FROM {arg})' + ) + + # Fix weekday CASE mapping after DAY_OF_WEEK → ISODOW conversion + # Old MySQL mapping: '2'→Monday, '3'→Tuesday...'1'→Sunday + # New ISODOW mapping: '1'→Monday, '2'→Tuesday...'7'→Sunday + weekday_replacements = { + "WHEN '2' THEN '1.Monday'": "WHEN '1' THEN '1.Monday'", + "WHEN '3' THEN '2.Tuesday'": "WHEN '2' THEN '2.Tuesday'", + "WHEN '4' THEN '3.Wednesday'": "WHEN '3' THEN '3.Wednesday'", + "WHEN '5' THEN '4.Thursday'": "WHEN '4' THEN '4.Thursday'", + "WHEN '6' THEN '5.Friday'": "WHEN '5' THEN '5.Friday'", + "WHEN '7' THEN '6.Saturday'": "WHEN '6' THEN '6.Saturday'", + "WHEN '1' THEN '7.Sunday'": "WHEN '7' THEN '7.Sunday'" + } + for old, new in weekday_replacements.items(): + sql = sql.replace(old, new) + + # DOUBLE PRECISION -> NUMERIC + # PostgreSQL ROUND() doesn't accept DOUBLE PRECISION, needs NUMERIC + # Simple global replacement since NUMERIC is compatible everywhere DOUBLE PRECISION is used + sql = re.sub( + r'\bDOUBLE\s+PRECISION\b', + 'NUMERIC', + sql, + flags=re.IGNORECASE + ) + + # NUMBER_TO_STR -> ROUND + cast to text + # MySQL custom: NUMBER_TO_STR(value, decimals) + # PostgreSQL: ROUND(value, decimals)::TEXT + sql = replace_function_with_balanced_parens( + sql, + r'\bNUMBER_TO_STR\s*\(', + lambda arg: f'ROUND({arg})::TEXT' + ) + + # FIND_IN_SET -> ANY(string_to_array()) + # MySQL: FIND_IN_SET(value, csv_string) > 0 + # PostgreSQL: value = ANY(string_to_array(csv_string, ',')) + # Iteratively replace innermost FIND_IN_SET first + prev_sql = None + while prev_sql != sql: + prev_sql = sql + sql = re.sub( + r'FIND_IN_SET\s*\(([^(),]+),\s*([^()]+)\)\s*>\s*0', + r"\1 = ANY(string_to_array(\2, ','))", + sql, + flags=re.IGNORECASE + ) + + # Fix malformed INTERVAL (missing number) + # INTERVAL DAY -> INTERVAL '1 day' + sql = re.sub( + r'\bINTERVAL\s+DAY\b', + "INTERVAL '1 day'", + sql, + flags=re.IGNORECASE + ) + + # ArgoCD images column is JSON - cast to text for GROUP BY/comparisons + # Pattern: ri.images (without ::text already) + # PostgreSQL JSONB requires explicit cast for equality/GROUP BY + sql = re.sub( + r'\bri\.images\b(?!\s*::)', + 'ri.images::text', + sql, + flags=re.IGNORECASE + ) + + # IFNULL -> COALESCE (if sqlglot missed it) + sql = re.sub(r'\bIFNULL\s*\(', 'COALESCE(', sql, flags=re.IGNORECASE) + + # UNIX_TIMESTAMP -> EXTRACT(EPOCH FROM ...) + sql = replace_function_with_balanced_parens( + sql, + r'\bUNIX_TIMESTAMP\s*\(', + lambda arg: f'EXTRACT(EPOCH FROM {arg})' + ) + + # TIMESTAMPDIFF - handle sqlglot format: TIMESTAMPDIFF(end, start, unit) + # PostgreSQL needs: EXTRACT(EPOCH FROM (end - start)) / divisor + # Use generic function to handle nested parens in arguments + def replace_timestampdiff(args): + # Parse comma-separated args manually (can't use split because of nested parens) + parts = [] + depth = 0 + current = [] + for char in args: + if char == '(': + depth += 1 + current.append(char) + elif char == ')': + depth -= 1 + current.append(char) + elif char == ',' and depth == 0: + parts.append(''.join(current).strip()) + current = [] + else: + current.append(char) + if current: + parts.append(''.join(current).strip()) + + if len(parts) >= 3: + end, start, unit = parts[0], parts[1], parts[2].upper() + # Map unit to divisor + divisors = {'DAY': '86400', 'HOUR': '3600', 'MINUTE': '60', 'SECOND': '1'} + divisor = divisors.get(unit, '1') + if divisor == '1': + return f'EXTRACT(EPOCH FROM ({end} - {start}))' + else: + return f'(EXTRACT(EPOCH FROM ({end} - {start}))/{divisor})' + return f'TIMESTAMPDIFF({args})' # Fallback if parse fails + + sql = replace_function_with_balanced_parens( + sql, + r'\bTIMESTAMPDIFF\s*\(', + replace_timestampdiff + ) + + # Step 3: Fix MySQL-specific type casts + sql = re.sub(r'\bUBIGINT\b', 'BIGINT', sql, flags=re.IGNORECASE) + sql = re.sub(r'\bAS\s+UNSIGNED\b', 'AS BIGINT', sql, flags=re.IGNORECASE) + sql = re.sub(r'\bAS\s+SIGNED\b', 'AS BIGINT', sql, flags=re.IGNORECASE) + + # Step 4: YEARWEEK() conversion + # YEARWEEK(col, 1) -> (EXTRACT(ISOYEAR FROM col) * 100 + EXTRACT(WEEK FROM col))::int + sql = re.sub( + r'\bYEARWEEK\s*\(\s*([^,]+?)\s*,\s*\d+\s*\)', + r'(EXTRACT(ISOYEAR FROM \1) * 100 + EXTRACT(WEEK FROM \1))::int', + sql, + flags=re.IGNORECASE + ) + + return sql + + +def protect_grafana_variables(sql: str) -> tuple: + """ + Replace Grafana template variables with safe placeholders before sqlglot. + sqlglot uppercases bare variables like $interval → $INTERVAL, breaking Grafana interpolation. + Protects both ${var} and $var (but not $__macros). + Returns: (modified_sql, dict of placeholders) + """ + placeholders = {} + counter = 0 + + def replacer(match): + nonlocal counter + placeholder = f'GRAFANA_PLACEHOLDER_{counter}_GRAFANA' + placeholders[placeholder] = match.group(0) + counter += 1 + return placeholder + + # Protect ${variable} or ${variable:format} + protected_sql = re.sub(r'\$\{([^}]+)\}', replacer, sql) + + # Protect bare $variable (but not $__macros like $__timeFilter) + protected_sql = re.sub(r'\$(?!__)[a-zA-Z_][a-zA-Z0-9_]*', replacer, protected_sql) + + return protected_sql, placeholders + + +def restore_grafana_variables(sql: str, placeholders: dict) -> str: + """ + Restore Grafana template variables from safe placeholders. + """ + for placeholder, original in placeholders.items(): + sql = sql.replace(placeholder, original) + + return sql + + +def restore_grafana_macros(sql: str) -> str: + """ + Restore Grafana macros to lowercase after sqlglot uppercases them. + Grafana expects: $__timeFilter(), $__timeFrom(), $__timeTo() + """ + sql = re.sub(r'\$__TIMEFILTER', '$__timeFilter', sql) + sql = re.sub(r'\$__TIMEFROM', '$__timeFrom', sql) + sql = re.sub(r'\$__TIMETO', '$__timeTo', sql) + + # Step 2: Fix WEEKDAY/INTERVAL conversion (broadened pattern) + # Handles: day - INTERVAL 'WEEKDAY DAY', metric_date - INTERVAL 'WEEKDAY DAY', + # TO_TIMESTAMP(...) - INTERVAL 'WEEKDAY DAY', etc. + # Find INTERVAL 'WEEKDAY DAY' and walk backwards to find expression + pattern = r"INTERVAL\s+['\"]WEEKDAY\s+DAY['\"]" + result = [] + i = 0 + + while i < len(sql): + match = re.search(pattern, sql[i:], re.IGNORECASE) + if not match: + result.append(sql[i:]) + break + + # Found INTERVAL 'WEEKDAY DAY' at position i + match.start() + interval_start = i + match.start() + + # Walk backwards to find the subtraction operator and LHS expression + # Look for ' - ' before the INTERVAL + minus_pos = None + for j in range(interval_start - 1, max(0, interval_start - 50), -1): + if sql[j:j+3] == ' - ' or (j > 0 and sql[j-1:j+2] == ' - '): + minus_pos = j + break + + if minus_pos is not None: + # Walk backwards from minus to find start of expression + # Stop at keywords, commas, or opening parens at depth 0 + expr_start = 0 + depth = 0 + for j in range(minus_pos - 1, -1, -1): + if sql[j] == ')': + depth += 1 + elif sql[j] == '(': + depth -= 1 + elif depth == 0 and sql[j] in (',', '\n'): + expr_start = j + 1 + break + elif depth == 0 and j >= 8: + # Check for SQL keywords - need to look for GROUP BY specifically + upper_substr = sql[max(0, j-8):j+1].upper() + for kw in ['GROUP BY ', 'ORDER BY ', 'SELECT ', 'WHERE ', 'AND ', 'WHEN ', 'BETWEEN ']: + kw_pos = upper_substr.rfind(kw) + if kw_pos >= 0: + # Found keyword, expr starts after it + expr_start = max(0, j-8) + kw_pos + len(kw) + break + if expr_start > 0: + break + + # Extract the expression + expr = sql[expr_start:minus_pos].strip() + + # Append everything before expr_start + result.append(sql[i:expr_start]) + + # Build replacement + replacement = f"{expr} - (EXTRACT(ISODOW FROM {expr}) - 1) * INTERVAL '1 day'" + result.append(replacement) + + # Skip past the INTERVAL 'WEEKDAY DAY' + # match.end() is relative to sql[i:], so absolute position is i + match.end() + i = i + match.end() + else: + # Could not find minus operator, keep original + result.append(match.group(0)) + i = i + match.end() + + sql = ''.join(result) + + # Also handle: expr + INTERVAL '6 DAY' pattern (week-end calculation) + # Convert: expr - INTERVAL 'WEEKDAY DAY' + INTERVAL '6 DAY' + # But this is handled naturally by the above since we only replace the WEEKDAY part + + return sql + + +def regex_fallback_conversion(sql: str) -> str: + """ + Regex-based fallback for SQL patterns sqlglot can't handle. + Handles nested functions and computed INTERVAL expressions. + """ + # CURDATE() -> CURRENT_DATE + sql = re.sub(r'\bCURDATE\s*\(\s*\)', 'CURRENT_DATE', sql, flags=re.IGNORECASE) + + # DATE(x) -> x::date + sql = re.sub(r'\bDATE\s*\(\s*([^)]+)\s*\)', r'(\1)::date', sql, flags=re.IGNORECASE) + + # DATE_FORMAT -> TO_CHAR (basic patterns) + sql = re.sub( + r"DATE_FORMAT\s*\(\s*([^,]+),\s*'%Y-%m-%d'\s*\)", + r"TO_CHAR(\1, 'YYYY-MM-DD')", + sql, + flags=re.IGNORECASE + ) + sql = re.sub( + r"DATE_FORMAT\s*\(\s*([^,]+),\s*'%Y/%m'\s*\)", + r"TO_CHAR(\1, 'YYYY/MM')", + sql, + flags=re.IGNORECASE + ) + sql = re.sub( + r"DATE_FORMAT\s*\(\s*([^,]+),\s*'%Y-%m'\s*\)", + r"TO_CHAR(\1, 'YYYY-MM')", + sql, + flags=re.IGNORECASE + ) + + # STR_TO_DATE -> TO_DATE + sql = re.sub(r'\bSTR_TO_DATE\s*\(', 'TO_DATE(', sql, flags=re.IGNORECASE) + + # CONVERT(expr, type) -> CAST(expr AS type) + sql = re.sub( + r'CONVERT\s*\(\s*([^,]+),\s*([^)]+)\)', + r'CAST(\1 AS \2)', + sql, + flags=re.IGNORECASE + ) + + # IF(cond, a, b) -> CASE WHEN cond THEN a ELSE b END + # Use balanced paren parsing to handle nested expressions + def replace_if(args): + # Parse comma-separated args manually (can't use split because of nested parens) + parts = [] + depth = 0 + current = [] + for char in args: + if char == ',' and depth == 0: + parts.append(''.join(current).strip()) + current = [] + else: + if char == '(': + depth += 1 + elif char == ')': + depth -= 1 + current.append(char) + if current: + parts.append(''.join(current).strip()) + + if len(parts) == 3: + return f'CASE WHEN {parts[0]} THEN {parts[1]} ELSE {parts[2]} END' + return f'IF({args})' # Fallback if parse fails + + sql = replace_function_with_balanced_parens( + sql, + r'\bIF\s*\(', + replace_if + ) + + # IFNULL -> COALESCE + sql = re.sub(r'\bIFNULL\s*\(', 'COALESCE(', sql, flags=re.IGNORECASE) + + # DATE_ADD/DATE_SUB with INTERVAL + # Handle: DATE_ADD(date, INTERVAL n DAY) -> date + INTERVAL 'n day' + sql = re.sub( + r'DATE_ADD\s*\(\s*([^,]+),\s*INTERVAL\s+([^)]+)\)', + r'\1 + INTERVAL ''\2''', + sql, + flags=re.IGNORECASE + ) + sql = re.sub( + r'DATE_SUB\s*\(\s*([^,]+),\s*INTERVAL\s+([^)]+)\)', + r'\1 - INTERVAL ''\2''', + sql, + flags=re.IGNORECASE + ) + + # Normalize INTERVAL syntax for PostgreSQL + sql = re.sub(r"INTERVAL\s+'(\d+)\s+DAY'", r"INTERVAL '\1 days'", sql, flags=re.IGNORECASE) + sql = re.sub(r"INTERVAL\s+'(\d+)\s+HOUR'", r"INTERVAL '\1 hours'", sql, flags=re.IGNORECASE) + + return sql + + +def process_dashboard_recursive(obj: Any, path: str = "") -> Any: + """ + Recursively process dashboard JSON, converting SQL and datasource references. + """ + if isinstance(obj, dict): + result = {} + for key, value in obj.items(): + current_path = f"{path}.{key}" if path else key + + # Convert rawSql fields + if key == "rawSql" and isinstance(value, str): + result[key] = convert_sql_mysql_to_postgres(value) + # Convert query and definition fields (dashboard variables) + elif key in ("query", "definition") and isinstance(value, str) and ("SELECT" in value.upper() or "CAST" in value.upper()): + result[key] = convert_sql_mysql_to_postgres(value) + # Convert datasource string references + elif key == "datasource" and isinstance(value, str) and value == "mysql": + result[key] = "postgresql" + # Convert datasource object references + elif key == "datasource" and isinstance(value, dict) and value.get("type") == "mysql": + result[key] = {**value, "type": "postgres"} + # Recursively process nested objects + else: + result[key] = process_dashboard_recursive(value, current_path) + + return result + + elif isinstance(obj, list): + return [process_dashboard_recursive(item, f"{path}[{i}]") for i, item in enumerate(obj)] + + else: + return obj + + +def convert_dashboard(input_path: Path, output_path: Path) -> None: + """ + Convert a single MySQL dashboard to PostgreSQL. + """ + try: + # Read input dashboard + with open(input_path, 'r', encoding='utf-8') as f: + dashboard = json.load(f) + + # Process dashboard recursively + converted = process_dashboard_recursive(dashboard) + + # Append -pg suffix to UID to avoid collisions + if "uid" in converted and converted["uid"]: + if not converted["uid"].endswith("-pg"): + converted["uid"] = f"{converted['uid']}-pg" + + # Title kept as-is (no PostgreSQL suffix needed) + + # Write output dashboard + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, 'w', encoding='utf-8') as f: + json.dump(converted, f, indent=2) + + print(f"✓ Converted: {input_path.name}") + + except json.JSONDecodeError as e: + print(f"✗ ERROR: Invalid JSON in {input_path}: {e}") + sys.exit(1) + except Exception as e: + print(f"✗ ERROR converting {input_path}: {e}") + sys.exit(1) + + +def main(): + if len(sys.argv) < 3: + print("Usage: python convert-mysql-to-postgresql.py ") + print(" input_path: MySQL dashboard JSON file or directory") + print(" output_path: PostgreSQL dashboard JSON file or directory") + sys.exit(1) + + input_path = Path(sys.argv[1]) + output_path = Path(sys.argv[2]) + + if not input_path.exists(): + print(f"ERROR: Input path does not exist: {input_path}") + sys.exit(1) + + # Single file conversion + if input_path.is_file(): + convert_dashboard(input_path, output_path) + + # Directory conversion + elif input_path.is_dir(): + json_files = list(input_path.glob("*.json")) + if not json_files: + print(f"WARNING: No JSON files found in {input_path}") + return + + print(f"Converting {len(json_files)} dashboards from {input_path} to {output_path}") + + for json_file in json_files: + output_file = output_path / json_file.name + convert_dashboard(json_file, output_file) + + print(f"\n✓ Conversion complete: {len(json_files)} dashboards") + + else: + print(f"ERROR: Input path is neither file nor directory: {input_path}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/grafana/scripts/entrypoint.sh b/grafana/scripts/entrypoint.sh new file mode 100644 index 00000000000..f5a13fa4c03 --- /dev/null +++ b/grafana/scripts/entrypoint.sh @@ -0,0 +1,244 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +DATASOURCE_FILE="/etc/grafana/provisioning/datasources/datasource.yml" + +# Detect database type +# Priority: DATABASE_TYPE, then legacy MYSQL_*/POSTGRES_* auto-detection +if [ -n "$DATABASE_TYPE" ]; then + MODE="$DATABASE_TYPE" + + case "$MODE" in + mysql) + export MYSQL_URL="${DATABASE_HOST:-mysql}:${DATABASE_PORT:-3306}" + export MYSQL_DATABASE="${DATABASE_NAME:-lake}" + export MYSQL_USER="${DATABASE_USER:-merico}" + export MYSQL_PASSWORD="${DATABASE_PASSWORD:-merico}" + ;; + postgresql) + export POSTGRES_URL="${DATABASE_HOST:-postgres}:${DATABASE_PORT:-5432}" + export POSTGRES_DATABASE="${DATABASE_NAME:-lake}" + export POSTGRES_USER="${DATABASE_USER:-merico}" + export POSTGRES_PASSWORD="${DATABASE_PASSWORD:-merico}" + ;; + *) + echo "ERROR: DATABASE_TYPE must be 'mysql' or 'postgresql'" + exit 1 + ;; + esac +else + # Legacy: auto-detect from MYSQL_*/POSTGRES_* vars + if [ -n "$POSTGRES_URL" ]; then + MODE="postgresql" + elif [ -n "$MYSQL_URL" ]; then + MODE="mysql" + else + echo "WARNING: No database vars. Defaulting to mysql." + MODE="mysql" + export MYSQL_URL="mysql:3306" + export MYSQL_DATABASE="lake" + export MYSQL_USER="merico" + export MYSQL_PASSWORD="merico" + fi +fi + +echo "Database type: $MODE" + +# Remove unused dashboard folder to prevent confusion +if [ "$MODE" = "mysql" ]; then + rm -rf /etc/grafana/dashboards/postgresql +else + rm -rf /etc/grafana/dashboards/mysql + SSL_MODE="${DATABASE_SSL_MODE:-disable}" + echo "SSL Mode: ${SSL_MODE}" +fi + +# Locate Homepage.json for the home dashboard. The layout differs by deployment: +# - baked-in image keeps variant subfolders: /etc/grafana/dashboards//Homepage.json +# - dev compose bind-mounts the variant folder directly: /etc/grafana/dashboards/Homepage.json +# Probe both so the home dashboard resolves in either case (a wrong path makes +# Grafana 12+ return HTTP 500 "Failed to load home dashboard"). +for _hp in \ + "/etc/grafana/dashboards/${MODE}/Homepage.json" \ + "/etc/grafana/dashboards/Homepage.json"; do + if [ -f "$_hp" ]; then + export GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH="$_hp" + break + fi +done + +echo "Homepage: $GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH" + +# --- Legacy persistent-volume hygiene ------------------------------------- +# This image runs as an arbitrary uid with gid 0 (OpenShift-compatible). Volumes +# created by older dashboard images (uid 472, file mode 0640) can be read-only for +# the current runtime user, which makes Grafana's DB migrations fail with +# "attempt to write a readonly database". Best-effort self-heal here; if it can't +# be fixed (non-root, non-owner) print a clear, actionable error instead of a +# cryptic SQLite crash. +GRAFANA_DATA_DIR="${GF_PATHS_DATA:-/var/lib/grafana}" +GRAFANA_DB="${GRAFANA_DATA_DIR}/grafana.db" + +if [ -e "$GRAFANA_DB" ] && [ ! -w "$GRAFANA_DB" ]; then + echo "WARNING: ${GRAFANA_DB} not writable by $(id -u):$(id -g); attempting permission fix..." + chgrp -R 0 "$GRAFANA_DATA_DIR" 2>/dev/null || true + chmod -R g+rwX "$GRAFANA_DATA_DIR" 2>/dev/null || true +fi + +if [ -e "$GRAFANA_DB" ] && [ ! -w "$GRAFANA_DB" ]; then + echo "ERROR: ${GRAFANA_DB} is still read-only for gid 0." + echo " The persistent volume was likely created by an older image (uid 472, mode 0640)." + echo " Fix it once from the host, then recreate this container:" + echo " docker run --rm -v _grafana-storage:/data alpine \\" + echo " sh -c 'chgrp -R 0 /data && chmod -R g+rwX /data'" +fi + +# Remove the deprecated Angular grafana-piechart-panel plugin if it lingers in a +# legacy volume. Grafana 12+ dropped Angular support and dashboards now use the +# core "piechart" panel; leaving it causes noisy plugin-validation errors. +LEGACY_PIECHART="${GRAFANA_DATA_DIR}/plugins/grafana-piechart-panel" +if [ -d "$LEGACY_PIECHART" ]; then + echo "Removing deprecated grafana-piechart-panel from data volume..." + rm -rf "$LEGACY_PIECHART" 2>/dev/null || true +fi +# -------------------------------------------------------------------------- + +# Create empty datasource.yml (datasources created via API) +cat > "$DATASOURCE_FILE" << HEADER +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +HEADER + +# Admin credentials used by the datasource provisioning API calls below. +# Defaults to Grafana's built-in admin password; override via GF_SECURITY_ADMIN_PASSWORD. +# Exported so Grafana itself and the curl calls use the same value. +export GF_SECURITY_ADMIN_PASSWORD="${GF_SECURITY_ADMIN_PASSWORD:-admin}" + +# Start Grafana in background +/run.sh "$@" & +GRAFANA_PID=$! + +# Wait for Grafana API +echo "Waiting for Grafana API..." +for i in $(seq 1 60); do + if curl -f -s http://localhost:3000/api/health >/dev/null 2>&1; then + echo "Grafana API ready" + sleep 5 # Extra wait for migrations + break + fi + sleep 2 +done + +# Grafana 13+ auto-creates empty datasource instances for built-in plugins and +# legacy volumes may contain read-only provisioned datasources with different UIDs. +# Delete ALL existing datasources (by ID, which bypasses read-only flags) so we can +# create a single correctly-configured datasource with the UID our dashboards expect. +echo "Deleting all existing datasources (clean slate)..." +for _id in $(curl -s "http://admin:${GF_SECURITY_ADMIN_PASSWORD}@localhost:3000/api/datasources" 2>/dev/null \ + | grep -o '"id":[0-9]*' | sed 's/"id"://g'); do + curl -s -X DELETE "http://admin:${GF_SECURITY_ADMIN_PASSWORD}@localhost:3000/api/datasources/${_id}" 2>&1 || true +done +sleep 2 + +# Create datasource via API (both MySQL and PostgreSQL) +PAYLOAD_FILE="/tmp/datasource-api.json" + +if [ "$MODE" = "mysql" ]; then + cat > "$PAYLOAD_FILE" < "$PAYLOAD_FILE" <&1) + + if echo "$RESPONSE" | grep -q '"id"'; then + echo "Datasource created successfully" + break + elif echo "$RESPONSE" | grep -q "already exists"; then + # A datasource with this name is already present (e.g. provisioned by an + # older image into a persistent volume). It cannot be recreated via the API + # but is functional, so treat this as success instead of retrying. + echo "Datasource already exists; keeping existing one" + break + elif echo "$RESPONSE" | grep -q "database is locked"; then + echo "DB locked, retry $i/10..." + sleep 3 + else + echo "API error: $RESPONSE" + sleep 2 + fi +done + +# Wait for Grafana +wait $GRAFANA_PID diff --git a/grafana/scripts/requirements.txt b/grafana/scripts/requirements.txt new file mode 100644 index 00000000000..fa90e1e7705 --- /dev/null +++ b/grafana/scripts/requirements.txt @@ -0,0 +1 @@ +sqlglot==30.12.0