From 414e473c5731b6dfce15386bdd30706eb05a2471 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Fri, 31 Jul 2026 15:34:09 +0200 Subject: [PATCH 01/10] feat: add github_snowflake plugin for Snowflake-backed GitHub ingestion Ingest GitHub PR data from Fivetran GITHUB_DB.MARTS into existing _tool_github_* tables and reuse adapted GitHub convertors, following the jira_snowflake pattern for per-repo historical sync without API rate limits. Co-Authored-By: Cursor Co-authored-by: Cursor --- backend/plugins/github_snowflake/AGENTS.md | 95 +++++++ .../github_snowflake/api/connection_api.go | 75 +++++ .../github_snowflake/github_snowflake.go | 43 +++ backend/plugins/github_snowflake/impl/impl.go | 224 +++++++++++++++ .../github_snowflake/models/connection.go | 51 ++++ .../models/migrationscripts/init_schema.go | 56 ++++ .../models/migrationscripts/register.go | 27 ++ .../tasks/convert_accounts.go | 150 ++++++++++ .../tasks/convert_pr_commits.go | 98 +++++++ .../tasks/convert_pr_reviews.go | 110 ++++++++ .../github_snowflake/tasks/convert_prs.go | 129 +++++++++ .../github_snowflake/tasks/convert_repo.go | 134 +++++++++ .../tasks/convert_reviewers.go | 100 +++++++ .../github_snowflake/tasks/main_test.go | 49 ++++ .../plugins/github_snowflake/tasks/shared.go | 53 ++++ .../github_snowflake/tasks/shared_test.go | 53 ++++ .../github_snowflake/tasks/sync_accounts.go | 152 ++++++++++ .../tasks/sync_accounts_test.go | 34 +++ .../github_snowflake/tasks/sync_pr_commits.go | 117 ++++++++ .../tasks/sync_pr_commits_test.go | 42 +++ .../github_snowflake/tasks/sync_pr_reviews.go | 128 +++++++++ .../tasks/sync_pr_reviews_test.go | 41 +++ .../tasks/sync_pull_requests.go | 180 ++++++++++++ .../tasks/sync_pull_requests_test.go | 56 ++++ .../github_snowflake/tasks/sync_repos.go | 109 ++++++++ .../github_snowflake/tasks/sync_repos_test.go | 32 +++ .../github_snowflake/tasks/sync_reviewers.go | 122 ++++++++ .../tasks/sync_reviewers_test.go | 42 +++ .../github_snowflake/tasks/task_data.go | 156 +++++++++++ .../github_snowflake/tasks/task_data_test.go | 71 +++++ backend/plugins/table_info_test.go | 2 + docs/github-snowflake-local-testing.md | 263 ++++++++++++++++++ docs/upstream-diffs.md | 5 +- 33 files changed, 2998 insertions(+), 1 deletion(-) create mode 100644 backend/plugins/github_snowflake/AGENTS.md create mode 100644 backend/plugins/github_snowflake/api/connection_api.go create mode 100644 backend/plugins/github_snowflake/github_snowflake.go create mode 100644 backend/plugins/github_snowflake/impl/impl.go create mode 100644 backend/plugins/github_snowflake/models/connection.go create mode 100644 backend/plugins/github_snowflake/models/migrationscripts/init_schema.go create mode 100644 backend/plugins/github_snowflake/models/migrationscripts/register.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_accounts.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_pr_commits.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_pr_reviews.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_prs.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_repo.go create mode 100644 backend/plugins/github_snowflake/tasks/convert_reviewers.go create mode 100644 backend/plugins/github_snowflake/tasks/main_test.go create mode 100644 backend/plugins/github_snowflake/tasks/shared.go create mode 100644 backend/plugins/github_snowflake/tasks/shared_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_accounts.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_accounts_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pr_commits.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pr_commits_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pr_reviews.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pr_reviews_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pull_requests.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_repos.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_repos_test.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_reviewers.go create mode 100644 backend/plugins/github_snowflake/tasks/sync_reviewers_test.go create mode 100644 backend/plugins/github_snowflake/tasks/task_data.go create mode 100644 backend/plugins/github_snowflake/tasks/task_data_test.go create mode 100644 docs/github-snowflake-local-testing.md diff --git a/backend/plugins/github_snowflake/AGENTS.md b/backend/plugins/github_snowflake/AGENTS.md new file mode 100644 index 00000000000..14231a64abf --- /dev/null +++ b/backend/plugins/github_snowflake/AGENTS.md @@ -0,0 +1,95 @@ +# github_snowflake Plugin — Agent Context + +Drop-in replacement for the GitHub API plugin per repository. Reads GitHub data +from a Snowflake replica (Fivetran, `GITHUB_DB.MARTS`) and writes into the +existing `_tool_github_*` tool-layer tables, then runs domain-layer convertors to +produce `code.*` domain records. + +## Build & Test + +```bash +cd backend +go build ./plugins/github_snowflake/... +go test ./plugins/github_snowflake/... -v +golangci-lint run ./plugins/github_snowflake/... +``` + +Unit tests do **not** need MySQL. A real pipeline run does. + +### Minimal real-run setup + +1. Start MySQL: `podman compose -f docker-compose-dev.yml up -d mysql` +2. Ensure `.env` has `DB_URL`, `ENCRYPTION_SECRET`, and (for local API) `AUTH_ENABLED=false` +3. Run the server (native desktop required for `externalbrowser` SSO): + +```bash +cd backend +DEVLAKE_PLUGINS=github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run +``` + +4. Create a connection + run a pipeline for one repo (`githubId` + `fullName`) + +Full step-by-step (Snowflake check, migrations, curl examples, result queries): +[docs/github-snowflake-local-testing.md](../../../docs/github-snowflake-local-testing.md) + +## Layout + +``` +impl/impl.go — plugin interfaces, SubTaskMetas, PrepareTaskData +api/connection_api.go — connection CRUD (POST/GET/PATCH/DELETE) +models/connection.go — SnowflakeGithubConnection (table: _tool_github_snowflake_connections) +models/migrationscripts/ — DB migrations +tasks/task_data.go — GithubSnowflakeOptions, GithubSnowflakeTaskData, OpenSnowflakeDB +tasks/shared.go — URL helpers +tasks/sync_*.go — Snowflake SQL queries → _tool_github_* tool-layer tables +tasks/convert_*.go — domain-layer convertors (adapted copies of github/tasks/*) +``` + +## Subtask pipeline order + +1. `syncRepos` — REPOSITORY → `_tool_github_repos` +2. `syncPullRequests` — PULL_REQUEST ⨝ ISSUE ⨝ ISSUE_MERGED ⨝ USER → `_tool_github_pull_requests` +3. `syncPrCommits` — COMMIT_PULL_REQUEST ⨝ COMMIT → `_tool_github_pull_request_commits` +4. `syncPrReviews` — PULL_REQUEST_REVIEW ⨝ USER → `_tool_github_pull_request_reviews` +5. `syncReviewers` — REQUESTED_REVIEWER_HISTORY → `_tool_github_reviewers` +6. `syncAccounts` — USER (+ USER_EMAIL) → `_tool_github_accounts` + `_tool_github_repo_accounts` +7. `convertRepo` / `convertPullRequests` / `convertPrCommits` / `convertPrReviews` / `convertReviews` / `convertAccounts` + +## Key conventions + +- **Scope unit is `GithubRepo`** (numeric `githubId` + `fullName` owner/repo). +- **No raw-table layer**: writes directly to `_tool_github_*`. Convertors use + `_raw_data_params`-scoped deletion for full sync (same pattern as jira_snowflake). +- **AuthType**: `"keypair"` (default, JWT) or `"externalbrowser"` (SSO, desktop only). +- **Connection defaults**: Database=`GITHUB_DB`, Schema=`MARTS`, Warehouse=`DEFAULT`. +- **PR line/comment counts** are unavailable in Snowflake — leave 0. +- **Actions jobs** table does not exist — never enable job sync/convertors. +- Do not log commit author emails in debug output. + +## Snowflake schema notes (GITHUB_DB.MARTS — verified 2026-07-21) + +| Table | Notes | +|---|---| +| `REPOSITORY` | No HTML/Clone URL — derive from FULL_NAME | +| `PULL_REQUEST` + `ISSUE` + `ISSUE_MERGED` | Fivetran splits PR fields across three tables | +| `COMMIT_PULL_REQUEST` | Orphan PR links exist — always INNER JOIN PULL_REQUEST | +| `PULL_REQUEST_REVIEW` | States: APPROVED, COMMENTED, DISMISSED, CHANGES_REQUESTED | +| `REQUESTED_REVIEWER_HISTORY` | Filter `REQUESTED_REVIEWER_TYPE = 'user'`; take latest non-removed | +| `"USER"` / `USER_EMAIL` | Identity present (not PII-stripped). Quote `"USER"` — reserved keyword in Snowflake. | + +Pilot coverage today: **konflux-ci** org only in MARTS. + +## GitHub plugin models dependency + +Imports `plugins/github/models` for tool-layer structs. This is a shared schema +dependency, not a business-logic cross-import. The github plugin does not need to +be deployed alongside github_snowflake — `impl.Init` registers a minimal +`githubPluginStub` for didgen. + +## Don'ts + +- Don't add models without a migration in `migrationscripts/register.go` +- Don't skip the Apache 2.0 license header on new `.go` files +- Don't configure the same repo in both a GitHub API connection and a + github_snowflake connection simultaneously — this causes domain ID duplication +- Don't enable Actions job convertors (table missing in Snowflake) diff --git a/backend/plugins/github_snowflake/api/connection_api.go b/backend/plugins/github_snowflake/api/connection_api.go new file mode 100644 index 00000000000..44faad79bde --- /dev/null +++ b/backend/plugins/github_snowflake/api/connection_api.go @@ -0,0 +1,75 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github_snowflake/models" +) + +var connHelper *helper.ConnectionApiHelper + +// Init initialises the API layer. Called from impl.Init. +func Init(br context.BasicRes, p plugin.PluginMeta) { + connHelper = helper.NewConnectionHelper(br, nil, p.Name()) +} + +// PostConnections creates a new SnowflakeGithubConnection. +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.SnowflakeGithubConnection{} + if err := connHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection, Status: 200}, nil +} + +// GetConnections returns all SnowflakeGithubConnections. +func GetConnections(_ *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.SnowflakeGithubConnection + if err := connHelper.List(&connections); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connections, Status: 200}, nil +} + +// GetConnection returns a single connection by connectionId path param. +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.SnowflakeGithubConnection{} + if err := connHelper.First(connection, input.Params); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection, Status: 200}, nil +} + +// PatchConnection updates fields on an existing connection. +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.SnowflakeGithubConnection{} + if err := connHelper.Patch(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection, Status: 200}, nil +} + +// DeleteConnection removes a connection. +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.SnowflakeGithubConnection{} + return connHelper.Delete(connection, input) +} diff --git a/backend/plugins/github_snowflake/github_snowflake.go b/backend/plugins/github_snowflake/github_snowflake.go new file mode 100644 index 00000000000..e9c3502b819 --- /dev/null +++ b/backend/plugins/github_snowflake/github_snowflake.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/github_snowflake/impl" + "github.com/spf13/cobra" +) + +// PluginEntry is exported for the DevLake framework to discover and load this plugin. +var PluginEntry impl.GithubSnowflake //nolint + +func main() { + cmd := &cobra.Command{Use: "github_snowflake"} + githubId := cmd.Flags().IntP("githubId", "g", 0, "GitHub repository numeric ID") + fullName := cmd.Flags().StringP("fullName", "n", "", "GitHub repository full name (owner/repo)") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "only sync records created/updated after this time (RFC3339)") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "githubId": *githubId, + "name": *fullName, + "fullName": *fullName, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/github_snowflake/impl/impl.go b/backend/plugins/github_snowflake/impl/impl.go new file mode 100644 index 00000000000..7d74be327c7 --- /dev/null +++ b/backend/plugins/github_snowflake/impl/impl.go @@ -0,0 +1,224 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coremodels "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/github_snowflake/api" + "github.com/apache/incubator-devlake/plugins/github_snowflake/models" + "github.com/apache/incubator-devlake/plugins/github_snowflake/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/github_snowflake/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginModel + plugin.PluginMigration + plugin.PluginApi + plugin.PluginSource + plugin.CloseablePluginTask +} = (*GithubSnowflake)(nil) + +// githubPluginStub is a minimal plugin.PluginMeta that lets didgen resolve +// types from plugins/github/models without requiring the full github plugin +// to be loaded alongside github_snowflake. +type githubPluginStub struct{} + +func (githubPluginStub) Name() string { return "github" } +func (githubPluginStub) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/github" +} +func (githubPluginStub) Description() string { return "" } + +// GithubSnowflake is the plugin implementation struct. +type GithubSnowflake struct{} + +func (p GithubSnowflake) Connection() dal.Tabler { + return &models.SnowflakeGithubConnection{} +} + +func (p GithubSnowflake) Scope() plugin.ToolLayerScope { + return &githubmodels.GithubRepo{} +} + +func (p GithubSnowflake) ScopeConfig() dal.Tabler { + return &githubmodels.GithubScopeConfig{} +} + +func (p GithubSnowflake) Init(basicRes context.BasicRes) errors.Error { + // Only register the github stub when the real github plugin is absent. + if _, err := plugin.GetPlugin("github"); err != nil { + _ = plugin.RegisterPlugin("github", githubPluginStub{}) + } + api.Init(basicRes, p) + return nil +} + +func (p GithubSnowflake) Name() string { + return "github_snowflake" +} + +func (p GithubSnowflake) Description() string { + return "Ingest GitHub data from a Snowflake replica (Fivetran) instead of the GitHub REST/GraphQL API" +} + +func (p GithubSnowflake) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/github_snowflake" +} + +func (p GithubSnowflake) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + // Sync tasks: Snowflake → _tool_github_* tables + tasks.SyncReposMeta, + tasks.SyncPullRequestsMeta, + tasks.SyncPrCommitsMeta, + tasks.SyncPrReviewsMeta, + tasks.SyncReviewersMeta, + tasks.SyncAccountsMeta, + // Convertor tasks: _tool_github_* → domain layer + tasks.ConvertRepoMeta, + tasks.ConvertPullRequestsMeta, + tasks.ConvertPullRequestCommitsMeta, + tasks.ConvertPullRequestReviewsMeta, + tasks.ConvertReviewsMeta, + tasks.ConvertAccountsMeta, + } +} + +func (p GithubSnowflake) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + + connection := &models.SnowflakeGithubConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "unable to get github_snowflake connection") + } + + // Ensure a GithubRepo scope record exists so convertors can find it. + repo := &githubmodels.GithubRepo{} + dbErr := taskCtx.GetDal().First(repo, dal.Where("connection_id = ? AND github_id = ?", op.ConnectionId, op.GithubId)) + if dbErr != nil && taskCtx.GetDal().IsErrorNotFound(dbErr) { + shortName := op.Name + if idx := len(op.Name) - 1; idx >= 0 { + for i := len(op.Name) - 1; i >= 0; i-- { + if op.Name[i] == '/' { + shortName = op.Name[i+1:] + break + } + } + } + repo = &githubmodels.GithubRepo{ + Scope: coremodels.Scope{ConnectionId: op.ConnectionId}, + GithubId: op.GithubId, + Name: shortName, + FullName: op.Name, + HTMLUrl: "https://github.com/" + op.Name, + CloneUrl: "https://github.com/" + op.Name + ".git", + } + if createErr := taskCtx.GetDal().CreateIfNotExist(repo); createErr != nil { + return nil, errors.Default.Wrap(createErr, "failed to create GithubRepo scope record") + } + } else if dbErr != nil { + return nil, errors.Default.Wrap(dbErr, "failed to look up GithubRepo scope record") + } else { + // Inherit name from existing scope if options omitted details. + if op.Name == "" && repo.FullName != "" { + op.Name = repo.FullName + op.FullName = repo.FullName + } + } + + if op.ScopeConfigId == 0 && repo.ScopeConfigId != 0 { + op.ScopeConfigId = repo.ScopeConfigId + } + if op.ScopeConfig == nil && op.ScopeConfigId != 0 { + var scopeConfig githubmodels.GithubScopeConfig + if loadErr := taskCtx.GetDal().First(&scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); loadErr != nil { + return nil, errors.BadInput.Wrap(loadErr, "failed to load scope config") + } + op.ScopeConfig = &scopeConfig + } + if op.ScopeConfig == nil { + op.ScopeConfig = new(githubmodels.GithubScopeConfig) + } + + snowDB, openErr := tasks.OpenSnowflakeDB( + connection.Account, + connection.User, + connection.AuthType, + connection.PrivateKey, + connection.Database, + connection.Schema, + connection.Warehouse, + connection.Role, + ) + if openErr != nil { + return nil, openErr + } + + return &tasks.GithubSnowflakeTaskData{ + Options: op, + SnowflakeDB: snowDB, + }, nil +} + +// Close is called after all subtasks complete; it closes the Snowflake connection. +func (p GithubSnowflake) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.GithubSnowflakeTaskData) + if ok && data != nil && data.SnowflakeDB != nil { + if err := data.SnowflakeDB.Close(); err != nil { + return errors.Default.Wrap(err, "failed to close Snowflake connection") + } + } + return nil +} + +func (p GithubSnowflake) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.SnowflakeGithubConnection{}, + } +} + +func (p GithubSnowflake) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p GithubSnowflake) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "connections": { + "POST": api.PostConnections, + "GET": api.GetConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + } +} diff --git a/backend/plugins/github_snowflake/models/connection.go b/backend/plugins/github_snowflake/models/connection.go new file mode 100644 index 00000000000..3770d8fa3b9 --- /dev/null +++ b/backend/plugins/github_snowflake/models/connection.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// SnowflakeGithubConnection holds the credentials and config for a Snowflake-backed GitHub connection. +// The private key PEM is stored encrypted (encrypt:"yes") using DevLake's field-level encryption. +type SnowflakeGithubConnection struct { + helper.BaseConnection `mapstructure:",squash"` + // Account is the Snowflake account identifier, e.g. "myorg-myaccount" + Account string `json:"account" gorm:"column:account;not null" mapstructure:"account" validate:"required"` + // User is the Snowflake service user name + User string `json:"user" gorm:"column:sf_user;not null" mapstructure:"user" validate:"required"` + // AuthType controls how the plugin authenticates to Snowflake. + // "keypair" (default/production): JWT authentication using PrivateKey. + // "externalbrowser": SSO via browser — only works when DevLake runs on a desktop host (not in a container). + AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` + // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. + // Required when AuthType is "keypair"; ignored for "externalbrowser". + PrivateKey string `json:"privateKey" encrypt:"yes" gorm:"column:private_key" mapstructure:"privateKey"` + // Database is the Snowflake database, e.g. "GITHUB_DB" + Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` + // Schema is the Snowflake schema, e.g. "MARTS" + Schema string `json:"schema" gorm:"column:sf_schema;not null" mapstructure:"schema" validate:"required"` + // Warehouse is the virtual warehouse to use; defaults to the account default + Warehouse string `json:"warehouse" gorm:"column:warehouse" mapstructure:"warehouse"` + // Role is the Snowflake role to assume + Role string `json:"role" gorm:"column:sf_role" mapstructure:"role"` +} + +func (SnowflakeGithubConnection) TableName() string { + return "_tool_github_snowflake_connections" +} diff --git a/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go b/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go new file mode 100644 index 00000000000..b7d6c94a689 --- /dev/null +++ b/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type initSchema struct{} + +// snowflakeGithubConnection20260731000001 is the schema snapshot at migration version 20260731000001. +type snowflakeGithubConnection20260731000001 struct { + helper.BaseConnection `mapstructure:",squash"` + Account string `gorm:"column:account;not null"` + User string `gorm:"column:sf_user;not null"` + AuthType string `gorm:"column:auth_type;default:keypair"` + PrivateKey string `gorm:"column:private_key"` + Database string `gorm:"column:sf_database;not null"` + Schema string `gorm:"column:sf_schema;not null"` + Warehouse string `gorm:"column:warehouse"` + Role string `gorm:"column:sf_role"` +} + +func (snowflakeGithubConnection20260731000001) TableName() string { + return "_tool_github_snowflake_connections" +} + +func (u *initSchema) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &snowflakeGithubConnection20260731000001{}) +} + +func (u *initSchema) Version() uint64 { + return 20260731000001 +} + +func (u *initSchema) Name() string { + return "github_snowflake init schema" +} diff --git a/backend/plugins/github_snowflake/models/migrationscripts/register.go b/backend/plugins/github_snowflake/models/migrationscripts/register.go new file mode 100644 index 00000000000..c0cd8affcfc --- /dev/null +++ b/backend/plugins/github_snowflake/models/migrationscripts/register.go @@ -0,0 +1,27 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import "github.com/apache/incubator-devlake/core/plugin" + +// All returns all migration scripts for the github_snowflake plugin. +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(initSchema), + } +} diff --git a/backend/plugins/github_snowflake/tasks/convert_accounts.go b/backend/plugins/github_snowflake/tasks/convert_accounts.go new file mode 100644 index 00000000000..d0fa529451f --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_accounts.go @@ -0,0 +1,150 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "convertAccounts", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer github_accounts into domain accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{ + githubmodels.GithubAccount{}.TableName(), + githubmodels.GithubRepoAccount{}.TableName(), + }, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + + accountIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubAccount{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubAccount]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_ACCOUNT_TABLE, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.Select("_tool_github_accounts.*"), + dal.From(&githubmodels.GithubAccount{}), + dal.Where( + "repo_github_id = ? and _tool_github_accounts.connection_id=?", + data.Options.GithubId, + data.Options.ConnectionId, + ), + dal.Join(`left join _tool_github_repo_accounts gra on ( + _tool_github_accounts.connection_id = gra.connection_id + AND _tool_github_accounts.id = gra.account_id + )`), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_accounts.updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubUser *githubmodels.GithubAccount) ([]interface{}, errors.Error) { + // Orgs are not synced from Snowflake in MVP — leave Organization empty. + domainUser := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{Id: accountIdGen.Generate(data.Options.ConnectionId, githubUser.Id)}, + Email: githubUser.Email, + FullName: githubUser.Name, + UserName: githubUser.Login, + AvatarUrl: githubUser.AvatarUrl, + } + return []interface{}{domainUser}, nil + }, + }) + if err != nil { + return err + } + + err = converter.Execute() + if err != nil { + return err + } + + return convertOrphanedRepoAccounts(taskCtx, db, data, accountIdGen) +} + +func convertOrphanedRepoAccounts(taskCtx plugin.SubTaskContext, db dal.Dal, data *GithubSnowflakeTaskData, accountIdGen *didgen.DomainIdGenerator) errors.Error { + cursor, err := db.Cursor( + dal.Select("_tool_github_repo_accounts.*"), + dal.From(&githubmodels.GithubRepoAccount{}), + dal.Where( + "_tool_github_repo_accounts.repo_github_id = ? AND _tool_github_repo_accounts.connection_id = ?", + data.Options.GithubId, + data.Options.ConnectionId, + ), + dal.Join(`LEFT JOIN _tool_github_accounts ga ON ( + _tool_github_repo_accounts.connection_id = ga.connection_id + AND _tool_github_repo_accounts.account_id = ga.id + )`), + dal.Where("ga.id IS NULL"), + ) + if err != nil { + return err + } + defer cursor.Close() + + logger := taskCtx.GetLogger() + for cursor.Next() { + var orphan githubmodels.GithubRepoAccount + err = db.Fetch(cursor, &orphan) + if err != nil { + return err + } + logger.Info("creating domain account for orphaned repo account: login=%s, id=%d", orphan.Login, orphan.AccountId) + domainUser := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, orphan.AccountId), + }, + UserName: orphan.Login, + FullName: orphan.Login, + } + err = db.CreateOrUpdate(domainUser) + if err != nil { + return err + } + } + if err := cursor.Err(); err != nil { + return errors.Default.Wrap(err, "iterating repo accounts cursor") + } + + return nil +} diff --git a/backend/plugins/github_snowflake/tasks/convert_pr_commits.go b/backend/plugins/github_snowflake/tasks/convert_pr_commits.go new file mode 100644 index 00000000000..f241562c116 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_pr_commits.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertPullRequestCommitsMeta = plugin.SubTaskMeta{ + Name: "convertPrCommits", + EntryPoint: ConvertPullRequestCommits, + EnabledByDefault: true, + Description: "Convert tool layer PR commits into domain pull_request_commits", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DependencyTables: []string{ + githubmodels.GithubPrCommit{}.TableName(), + githubmodels.GithubPullRequest{}.TableName(), + }, + ProductTables: []string{code.PullRequestCommit{}.TableName()}, +} + +func ConvertPullRequestCommits(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + logger := taskCtx.GetLogger() + repoId := data.Options.GithubId + + pullIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubPullRequest{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubPrCommit]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_COMMIT_TABLE, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&githubmodels.GithubPrCommit{}), + dal.Join(`left join _tool_github_pull_requests on _tool_github_pull_requests.github_id = _tool_github_pull_request_commits.pull_request_id AND _tool_github_pull_requests.connection_id = _tool_github_pull_request_commits.connection_id`), + dal.Where("_tool_github_pull_requests.repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + dal.Orderby("pull_request_id ASC"), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPullRequestCommit *githubmodels.GithubPrCommit) ([]interface{}, errors.Error) { + domainPrCommit := &code.PullRequestCommit{ + CommitSha: githubPullRequestCommit.CommitSha, + PullRequestId: pullIdGen.Generate(data.Options.ConnectionId, githubPullRequestCommit.PullRequestId), + CommitAuthorName: githubPullRequestCommit.CommitAuthorName, + CommitAuthorEmail: githubPullRequestCommit.CommitAuthorEmail, + CommitAuthoredDate: githubPullRequestCommit.CommitAuthoredDate, + } + return []interface{}{domainPrCommit}, nil + }, + }) + if err != nil { + return err + } + + if !converter.IsIncremental() { + logger.Debug("deleting outdated domain pull_request_commits for repo %d", repoId) + if dbErr := db.Delete(&code.PullRequestCommit{}, dal.Where("_raw_data_params = ?", converter.GetRawDataParams())); dbErr != nil { + return dbErr + } + } + + return converter.Execute() +} diff --git a/backend/plugins/github_snowflake/tasks/convert_pr_reviews.go b/backend/plugins/github_snowflake/tasks/convert_pr_reviews.go new file mode 100644 index 00000000000..e4e125c9df9 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_pr_reviews.go @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertPullRequestReviewsMeta = plugin.SubTaskMeta{ + Name: "convertPrReviews", + EntryPoint: ConvertPullRequestReviews, + EnabledByDefault: true, + Description: "Convert tool layer PR reviews into domain pull_request_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + DependencyTables: []string{ + githubmodels.GithubPrReview{}.TableName(), + githubmodels.GithubPullRequest{}.TableName(), + githubmodels.GithubAccount{}.TableName(), + }, + ProductTables: []string{code.PullRequestComment{}.TableName()}, +} + +func ConvertPullRequestReviews(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + logger := taskCtx.GetLogger() + repoId := data.Options.GithubId + + prReviewUIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubPrReview{}) + prIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubPullRequest{}) + accountIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubAccount{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubPrReview]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_REVIEW_TABLE, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&githubmodels.GithubPrReview{}), + dal.Join("left join _tool_github_pull_requests " + + "on _tool_github_pull_requests.github_id = _tool_github_pull_request_reviews.pull_request_id " + + "and _tool_github_pull_requests.connection_id = _tool_github_pull_request_reviews.connection_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubPullRequestReview *githubmodels.GithubPrReview) ([]interface{}, errors.Error) { + domainPrReview := &code.PullRequestComment{ + DomainEntity: domainlayer.DomainEntity{ + Id: prReviewUIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.GithubId), + }, + PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.PullRequestId), + Body: githubPullRequestReview.Body, + AccountId: accountIdGen.Generate(data.Options.ConnectionId, githubPullRequestReview.AuthorUserId), + CommitSha: githubPullRequestReview.CommitSha, + Type: "REVIEW", + Status: githubPullRequestReview.State, + } + if githubPullRequestReview.GithubSubmitAt != nil { + domainPrReview.CreatedDate = *githubPullRequestReview.GithubSubmitAt + } + return []interface{}{domainPrReview}, nil + }, + }) + if err != nil { + return err + } + + if !converter.IsIncremental() { + logger.Debug("deleting outdated domain pull_request_comments (reviews) for repo %d", repoId) + if dbErr := db.Delete(&code.PullRequestComment{}, dal.Where("_raw_data_params = ? AND type = ?", converter.GetRawDataParams(), "REVIEW")); dbErr != nil { + return dbErr + } + } + + return converter.Execute() +} diff --git a/backend/plugins/github_snowflake/tasks/convert_prs.go b/backend/plugins/github_snowflake/tasks/convert_prs.go new file mode 100644 index 00000000000..4a01fc1d82c --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_prs.go @@ -0,0 +1,129 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertPullRequestsMeta = plugin.SubTaskMeta{ + Name: "convertPullRequests", + EntryPoint: ConvertPullRequests, + EnabledByDefault: true, + Description: "Convert tool layer pull requests into domain layer pull_requests", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE_REVIEW}, + DependencyTables: []string{ + githubmodels.GithubPullRequest{}.TableName(), + githubmodels.GithubAccount{}.TableName(), + }, + ProductTables: []string{code.PullRequest{}.TableName()}, +} + +func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + logger := taskCtx.GetLogger() + repoId := data.Options.GithubId + + prIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubPullRequest{}) + repoIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubRepo{}) + accountIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubAccount{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubPullRequest]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PULL_REQUEST_TABLE, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&githubmodels.GithubPullRequest{}), + dal.Where("repo_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(pr *githubmodels.GithubPullRequest) ([]interface{}, errors.Error) { + domainPr := &code.PullRequest{ + DomainEntity: domainlayer.DomainEntity{ + Id: prIdGen.Generate(data.Options.ConnectionId, pr.GithubId), + }, + BaseRepoId: repoIdGen.Generate(data.Options.ConnectionId, pr.RepoId), + HeadRepoId: repoIdGen.Generate(data.Options.ConnectionId, pr.HeadRepoId), + OriginalStatus: pr.State, + Title: pr.Title, + Url: pr.Url, + AuthorId: accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId), + AuthorName: pr.AuthorName, + Description: pr.Body, + CreatedDate: pr.GithubCreatedAt, + MergedDate: pr.MergedAt, + ClosedDate: pr.ClosedAt, + PullRequestKey: pr.Number, + Type: pr.Type, + Component: pr.Component, + MergeCommitSha: pr.MergeCommitSha, + BaseRef: pr.BaseRef, + BaseCommitSha: pr.BaseCommitSha, + HeadRef: pr.HeadRef, + HeadCommitSha: pr.HeadCommitSha, + Additions: pr.Additions, + Deletions: pr.Deletions, + MergedByName: pr.MergedByName, + MergedById: accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById), + IsDraft: pr.IsDraft, + } + if pr.State == "open" || pr.State == "OPEN" { + domainPr.Status = code.OPEN + } else if pr.State == "MERGED" || (pr.State == "closed" && (pr.Merged || pr.MergedAt != nil)) { + domainPr.Status = code.MERGED + } else { + domainPr.Status = code.CLOSED + } + return []interface{}{domainPr}, nil + }, + }) + if err != nil { + return err + } + + // Full-sync deletion: no raw-table layer, so delete by _raw_data_params before convert. + if !converter.IsIncremental() { + logger.Debug("deleting outdated domain pull_requests for repo %d", repoId) + if dbErr := db.Delete(&code.PullRequest{}, dal.Where("_raw_data_params = ?", converter.GetRawDataParams())); dbErr != nil { + return dbErr + } + } + + return converter.Execute() +} diff --git a/backend/plugins/github_snowflake/tasks/convert_repo.go b/backend/plugins/github_snowflake/tasks/convert_repo.go new file mode 100644 index 00000000000..7e3bbe84f12 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_repo.go @@ -0,0 +1,134 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertRepoMeta = plugin.SubTaskMeta{ + Name: "convertRepo", + EntryPoint: ConvertRepo, + EnabledByDefault: true, + Description: "Convert tool layer table github_repos into domain layer table repos and boards", + DomainTypes: []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_TICKET, + plugin.DOMAIN_TYPE_CICD, + plugin.DOMAIN_TYPE_CODE_REVIEW, + plugin.DOMAIN_TYPE_CROSS, + }, + ProductTables: []string{ + code.Repo{}.TableName(), + ticket.Board{}.TableName(), + crossdomain.BoardRepo{}.TableName(), + devops.CicdScope{}.TableName(), + }, +} + +func ConvertRepo(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + repoId := data.Options.GithubId + + repoIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubRepo{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubRepo]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: githubmodels.GithubRepo{}.TableName(), + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&githubmodels.GithubRepo{}), + dal.Where("github_id = ? and connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(repository *githubmodels.GithubRepo) ([]interface{}, errors.Error) { + domainRepository := &code.Repo{ + DomainEntity: domainlayer.DomainEntity{ + Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), + }, + Name: repository.FullName, + Url: repository.HTMLUrl, + Description: repository.Description, + ForkedFrom: repository.ParentHTMLUrl, + Language: repository.Language, + CreatedDate: repository.CreatedDate, + UpdatedDate: repository.UpdatedDate, + } + domainBoard := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), + }, + Name: repository.FullName, + Url: fmt.Sprintf("%s/%s", repository.HTMLUrl, "issues"), + Description: repository.Description, + CreatedDate: repository.CreatedDate, + } + domainBoardRepo := &crossdomain.BoardRepo{ + BoardId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), + RepoId: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), + } + domainCicdScope := &devops.CicdScope{ + DomainEntity: domainlayer.DomainEntity{ + Id: repoIdGen.Generate(data.Options.ConnectionId, repository.GithubId), + }, + Name: repository.FullName, + Url: repository.HTMLUrl, + Description: repository.Description, + CreatedDate: repository.CreatedDate, + UpdatedDate: repository.UpdatedDate, + } + return []interface{}{ + domainRepository, + domainBoard, + domainBoardRepo, + domainCicdScope, + }, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/github_snowflake/tasks/convert_reviewers.go b/backend/plugins/github_snowflake/tasks/convert_reviewers.go new file mode 100644 index 00000000000..f9383fbb0c1 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/convert_reviewers.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var ConvertReviewsMeta = plugin.SubTaskMeta{ + Name: "convertReviews", + EntryPoint: ConvertReviews, + EnabledByDefault: true, + Description: "Convert tool layer reviewers into domain pull_request_reviewers", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, + DependencyTables: []string{ + githubmodels.GithubReviewer{}.TableName(), + githubmodels.GithubPullRequest{}.TableName(), + githubmodels.GithubAccount{}.TableName(), + }, + ProductTables: []string{code.PullRequestReviewer{}.TableName()}, +} + +func ConvertReviews(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*GithubSnowflakeTaskData) + logger := taskCtx.GetLogger() + repoId := data.Options.GithubId + + prIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubPullRequest{}) + accountIdGen := didgen.NewDomainIdGenerator(&githubmodels.GithubAccount{}) + + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[githubmodels.GithubReviewer]{ + SubtaskCommonArgs: &api.SubtaskCommonArgs{ + SubTaskContext: taskCtx, + Table: RAW_PR_REVIEW_TABLE, + Params: GithubApiParams{ + ConnectionId: data.Options.ConnectionId, + Name: data.Options.Name, + }, + }, + Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { + clauses := []dal.Clause{ + dal.From(&githubmodels.GithubReviewer{}), + dal.Join("left join _tool_github_pull_requests " + + "on _tool_github_pull_requests.github_id = _tool_github_reviewers.pull_request_id " + + "and _tool_github_pull_requests.connection_id = _tool_github_reviewers.connection_id"), + dal.Where("repo_id = ? and _tool_github_pull_requests.connection_id = ?", repoId, data.Options.ConnectionId), + } + if stateManager.IsIncremental() { + since := stateManager.GetSince() + if since != nil { + clauses = append(clauses, dal.Where("_tool_github_pull_requests.github_updated_at >= ?", since)) + } + } + return db.Cursor(clauses...) + }, + Convert: func(githubReview *githubmodels.GithubReviewer) ([]interface{}, errors.Error) { + domainReview := &code.PullRequestReviewer{ + PullRequestId: prIdGen.Generate(data.Options.ConnectionId, githubReview.PullRequestId), + ReviewerId: accountIdGen.Generate(data.Options.ConnectionId, githubReview.ReviewerId), + Name: githubReview.Name, + UserName: githubReview.Username, + } + return []interface{}{domainReview}, nil + }, + }) + if err != nil { + return err + } + + if !converter.IsIncremental() { + logger.Debug("deleting outdated domain pull_request_reviewers for repo %d", repoId) + if dbErr := db.Delete(&code.PullRequestReviewer{}, dal.Where("_raw_data_params = ?", converter.GetRawDataParams())); dbErr != nil { + return dbErr + } + } + + return converter.Execute() +} diff --git a/backend/plugins/github_snowflake/tasks/main_test.go b/backend/plugins/github_snowflake/tasks/main_test.go new file mode 100644 index 00000000000..396bccc072e --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/main_test.go @@ -0,0 +1,49 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "testing" + + "github.com/apache/incubator-devlake/core/plugin" +) + +func TestMain(m *testing.M) { + _ = plugin.RegisterPlugin("github", githubPluginStub{}) + _ = plugin.RegisterPlugin("github_snowflake", GithubSnowflake{}) + os.Exit(m.Run()) +} + +// githubPluginStub lets didgen resolve plugins/github/models types in unit tests. +type githubPluginStub struct{} + +func (githubPluginStub) Name() string { return "github" } +func (githubPluginStub) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/github" +} +func (githubPluginStub) Description() string { return "" } + +// GithubSnowflake is a minimal PluginMeta stub for github_snowflake model types. +type GithubSnowflake struct{} + +func (GithubSnowflake) Name() string { return "github_snowflake" } +func (GithubSnowflake) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/github_snowflake" +} +func (GithubSnowflake) Description() string { return "" } diff --git a/backend/plugins/github_snowflake/tasks/shared.go b/backend/plugins/github_snowflake/tasks/shared.go new file mode 100644 index 00000000000..301aa4bbe67 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/shared.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" +) + +// deriveRepoHTMLUrl builds the GitHub HTML URL from FULL_NAME. +func deriveRepoHTMLUrl(fullName string) string { + return fmt.Sprintf("https://github.com/%s", fullName) +} + +// deriveRepoCloneUrl builds the GitHub clone URL from FULL_NAME. +func deriveRepoCloneUrl(fullName string) string { + return fmt.Sprintf("https://github.com/%s.git", fullName) +} + +// derivePullRequestURL builds the GitHub PR URL. +func derivePullRequestURL(fullName string, number int) string { + return fmt.Sprintf("https://github.com/%s/pull/%d", fullName, number) +} + +// nullStr returns the string value or empty string when nil. +func nullStr(s *string) string { + if s == nil { + return "" + } + return *s +} + +// nullInt returns the int value or 0 when nil. +func nullInt(i *int64) int { + if i == nil { + return 0 + } + return int(*i) +} diff --git a/backend/plugins/github_snowflake/tasks/shared_test.go b/backend/plugins/github_snowflake/tasks/shared_test.go new file mode 100644 index 00000000000..54849817e93 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/shared_test.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDeriveRepoHTMLUrl(t *testing.T) { + assert.Equal(t, "https://github.com/konflux-ci/build-service", deriveRepoHTMLUrl("konflux-ci/build-service")) +} + +func TestDeriveRepoCloneUrl(t *testing.T) { + assert.Equal(t, "https://github.com/konflux-ci/build-service.git", deriveRepoCloneUrl("konflux-ci/build-service")) +} + +func TestDerivePullRequestURL(t *testing.T) { + assert.Equal(t, "https://github.com/konflux-ci/build-service/pull/42", derivePullRequestURL("konflux-ci/build-service", 42)) +} + +func TestNullStr(t *testing.T) { + assert.Equal(t, "", nullStr(nil)) + s := "hello" + assert.Equal(t, "hello", nullStr(&s)) +} + +func TestNullInt(t *testing.T) { + assert.Equal(t, 0, nullInt(nil)) + v := int64(99) + assert.Equal(t, 99, nullInt(&v)) +} + +func TestRepoShortName(t *testing.T) { + assert.Equal(t, "build-service", repoShortName("konflux-ci/build-service")) + assert.Equal(t, "alone", repoShortName("alone")) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_accounts.go b/backend/plugins/github_snowflake/tasks/sync_accounts.go new file mode 100644 index 00000000000..07afdea070a --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_accounts.go @@ -0,0 +1,152 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncAccountsMeta = plugin.SubTaskMeta{ + Name: "syncAccounts", + EntryPoint: SyncAccounts, + EnabledByDefault: true, + Description: "Sync GitHub users referenced by repo PRs/reviews from Snowflake into _tool_github_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +func SyncAccounts(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + repoId := data.Options.GithubId + + query, args := buildAccountsQuery(repoId) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for accounts") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + id int64 + login *string + name *string + email *string + ) + if scanErr := rows.Scan(&id, &login, &name, &email); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan account row") + } + + loginStr := nullStr(login) + account := &githubmodels.GithubAccount{ + ConnectionId: connectionId, + Id: int(id), + Login: loginStr, + Name: nullStr(name), + Email: nullStr(email), + NoPKModel: common.NewNoPKModel(), + } + if loginStr != "" { + account.HtmlUrl = "https://github.com/" + loginStr + account.Url = "https://api.github.com/users/" + loginStr + } + if dbErr := db.CreateOrUpdate(account); dbErr != nil { + return dbErr + } + + repoAccount := &githubmodels.GithubRepoAccount{ + ConnectionId: connectionId, + AccountId: int(id), + RepoGithubId: repoId, + Login: loginStr, + NoPKModel: common.NewNoPKModel(), + } + if dbErr := db.CreateOrUpdate(repoAccount); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating account rows") + } + + logger.Info("synced %d accounts from Snowflake", count) + return nil +} + +func buildAccountsQuery(repoId int) (string, []interface{}) { + // Distinct users referenced as PR authors, mergers, review authors, or requested reviewers. + query := ` +WITH repo_users AS ( + SELECT i.USER_ID AS user_id + FROM PULL_REQUEST pr + JOIN ISSUE i ON i.ID = pr.ISSUE_ID + WHERE i.REPOSITORY_ID = ? + AND i.USER_ID IS NOT NULL + + UNION + + SELECT im.ACTOR_ID AS user_id + FROM PULL_REQUEST pr + JOIN ISSUE i ON i.ID = pr.ISSUE_ID + JOIN ISSUE_MERGED im ON im.ISSUE_ID = i.ID + WHERE i.REPOSITORY_ID = ? + AND im.ACTOR_ID IS NOT NULL + + UNION + + SELECT r.USER_ID AS user_id + FROM PULL_REQUEST_REVIEW r + JOIN PULL_REQUEST pr ON pr.ID = r.PULL_REQUEST_ID + JOIN ISSUE i ON i.ID = pr.ISSUE_ID + WHERE i.REPOSITORY_ID = ? + AND r.USER_ID IS NOT NULL + + UNION + + SELECT h.REQUESTED_ID AS user_id + FROM REQUESTED_REVIEWER_HISTORY h + JOIN PULL_REQUEST pr ON pr.ID = h.PULL_REQUEST_ID + JOIN ISSUE i ON i.ID = pr.ISSUE_ID + WHERE i.REPOSITORY_ID = ? + AND LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user' + AND h.REQUESTED_ID IS NOT NULL +) +SELECT + u.ID, + u.LOGIN, + u.NAME, + ue.EMAIL +FROM "USER" u +JOIN repo_users ru ON ru.user_id = u.ID +LEFT JOIN USER_EMAIL ue + ON ue.USER_ID = u.ID +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY u.ID + ORDER BY ue.EMAIL NULLS LAST +) = 1 +` + return query, []interface{}{repoId, repoId, repoId, repoId} +} diff --git a/backend/plugins/github_snowflake/tasks/sync_accounts_test.go b/backend/plugins/github_snowflake/tasks/sync_accounts_test.go new file mode 100644 index 00000000000..c457bb671bd --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_accounts_test.go @@ -0,0 +1,34 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildAccountsQuery(t *testing.T) { + q, args := buildAccountsQuery(55) + assert.Contains(t, q, `FROM "USER" u`) + assert.Contains(t, q, "LEFT JOIN USER_EMAIL ue") + assert.Contains(t, q, "WITH repo_users AS") + assert.Contains(t, q, "PULL_REQUEST_REVIEW") + assert.Contains(t, q, "REQUESTED_REVIEWER_HISTORY") + assert.Equal(t, []interface{}{55, 55, 55, 55}, args) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_commits.go b/backend/plugins/github_snowflake/tasks/sync_pr_commits.go new file mode 100644 index 00000000000..e8df6412a65 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pr_commits.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncPrCommitsMeta = plugin.SubTaskMeta{ + Name: "syncPrCommits", + EntryPoint: SyncPrCommits, + EnabledByDefault: true, + Description: "Sync GitHub PR commits from Snowflake into _tool_github_pull_request_commits", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS}, +} + +func SyncPrCommits(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + repoId := data.Options.GithubId + + var timeAfter *time.Time + syncPolicy := subtaskCtx.TaskContext().SyncPolicy() + if syncPolicy != nil { + timeAfter = syncPolicy.TimeAfter + } + + query, args := buildPrCommitsQuery(repoId, timeAfter) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for PR commits") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + commitSha string + pullRequestId int64 + authorName *string + authorEmail *string + authoredDate time.Time + ) + if scanErr := rows.Scan(&commitSha, &pullRequestId, &authorName, &authorEmail, &authoredDate); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan PR commit row") + } + + prCommit := &githubmodels.GithubPrCommit{ + ConnectionId: connectionId, + CommitSha: commitSha, + PullRequestId: int(pullRequestId), + CommitAuthorName: nullStr(authorName), + CommitAuthorEmail: nullStr(authorEmail), + CommitAuthoredDate: authoredDate, + NoPKModel: common.NewNoPKModel(), + } + if dbErr := db.CreateOrUpdate(prCommit); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating PR commit rows") + } + + logger.Info("synced %d PR commits from Snowflake", count) + return nil +} + +func buildPrCommitsQuery(repoId int, timeAfter *time.Time) (string, []interface{}) { + // INNER JOIN PULL_REQUEST drops orphaned historical COMMIT_PULL_REQUEST links. + query := ` +SELECT + cpr.COMMIT_SHA, + cpr.PULL_REQUEST_ID, + c.AUTHOR_NAME, + c.AUTHOR_EMAIL, + c.AUTHOR_DATE +FROM COMMIT_PULL_REQUEST cpr +JOIN COMMIT c + ON c.SHA = cpr.COMMIT_SHA +JOIN PULL_REQUEST pr + ON pr.ID = cpr.PULL_REQUEST_ID +JOIN ISSUE i + ON i.ID = pr.ISSUE_ID +WHERE i.REPOSITORY_ID = ? +` + args := []interface{}{repoId} + if timeAfter != nil { + query += " AND c.AUTHOR_DATE > ?\n" + args = append(args, *timeAfter) + } + return query, args +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_commits_test.go b/backend/plugins/github_snowflake/tasks/sync_pr_commits_test.go new file mode 100644 index 00000000000..6bceb5e986e --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pr_commits_test.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPrCommitsQuery_InnerJoinsPullRequest(t *testing.T) { + q, args := buildPrCommitsQuery(99, nil) + assert.Contains(t, q, "JOIN PULL_REQUEST pr") + assert.Contains(t, q, "JOIN COMMIT c") + assert.Contains(t, q, "FROM COMMIT_PULL_REQUEST cpr") + assert.Contains(t, q, "WHERE i.REPOSITORY_ID = ?") + assert.NotContains(t, q, "AUTHOR_DATE >") + assert.Equal(t, []interface{}{99}, args) +} + +func TestBuildPrCommitsQuery_WithTimeFilter(t *testing.T) { + ts := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC) + q, args := buildPrCommitsQuery(99, &ts) + assert.Contains(t, q, "AND c.AUTHOR_DATE > ?") + assert.Equal(t, ts, args[1]) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go b/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go new file mode 100644 index 00000000000..8438908c612 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go @@ -0,0 +1,128 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncPrReviewsMeta = plugin.SubTaskMeta{ + Name: "syncPrReviews", + EntryPoint: SyncPrReviews, + EnabledByDefault: true, + Description: "Sync GitHub PR reviews from Snowflake into _tool_github_pull_request_reviews", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, +} + +func SyncPrReviews(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + repoId := data.Options.GithubId + + var timeAfter *time.Time + syncPolicy := subtaskCtx.TaskContext().SyncPolicy() + if syncPolicy != nil { + timeAfter = syncPolicy.TimeAfter + } + + query, args := buildPrReviewsQuery(repoId, timeAfter) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for PR reviews") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + githubId int64 + pullRequestId int64 + body *string + state *string + commitSha *string + submittedAt *time.Time + authorUserId *int64 + authorLogin *string + ) + if scanErr := rows.Scan( + &githubId, &pullRequestId, &body, &state, &commitSha, + &submittedAt, &authorUserId, &authorLogin, + ); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan PR review row") + } + + review := &githubmodels.GithubPrReview{ + ConnectionId: connectionId, + GithubId: int(githubId), + PullRequestId: int(pullRequestId), + Body: nullStr(body), + State: nullStr(state), + CommitSha: nullStr(commitSha), + GithubSubmitAt: submittedAt, + AuthorUserId: nullInt(authorUserId), + AuthorUsername: nullStr(authorLogin), + NoPKModel: common.NewNoPKModel(), + } + if dbErr := db.CreateOrUpdate(review); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating PR review rows") + } + + logger.Info("synced %d PR reviews from Snowflake", count) + return nil +} + +func buildPrReviewsQuery(repoId int, timeAfter *time.Time) (string, []interface{}) { + query := ` +SELECT + r.ID AS github_id, + r.PULL_REQUEST_ID AS pull_request_id, + r.BODY, + r.STATE, + r.COMMIT_SHA, + r.SUBMITTED_AT, + r.USER_ID AS author_user_id, + u.LOGIN AS author_username +FROM PULL_REQUEST_REVIEW r +JOIN PULL_REQUEST pr + ON pr.ID = r.PULL_REQUEST_ID +JOIN ISSUE i + ON i.ID = pr.ISSUE_ID +LEFT JOIN "USER" u + ON u.ID = r.USER_ID +WHERE i.REPOSITORY_ID = ? +` + args := []interface{}{repoId} + if timeAfter != nil { + query += " AND r.SUBMITTED_AT > ?\n" + args = append(args, *timeAfter) + } + return query, args +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_reviews_test.go b/backend/plugins/github_snowflake/tasks/sync_pr_reviews_test.go new file mode 100644 index 00000000000..8a1df08cd19 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pr_reviews_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPrReviewsQuery_NoTimeFilter(t *testing.T) { + q, args := buildPrReviewsQuery(7, nil) + assert.Contains(t, q, "FROM PULL_REQUEST_REVIEW r") + assert.Contains(t, q, `LEFT JOIN "USER" u`) + assert.Contains(t, q, "WHERE i.REPOSITORY_ID = ?") + assert.NotContains(t, q, "SUBMITTED_AT >") + assert.Equal(t, []interface{}{7}, args) +} + +func TestBuildPrReviewsQuery_WithTimeFilter(t *testing.T) { + ts := time.Date(2026, 4, 1, 0, 0, 0, 0, time.UTC) + q, args := buildPrReviewsQuery(7, &ts) + assert.Contains(t, q, "AND r.SUBMITTED_AT > ?") + assert.Equal(t, ts, args[1]) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go new file mode 100644 index 00000000000..1eac8838436 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go @@ -0,0 +1,180 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncPullRequestsMeta = plugin.SubTaskMeta{ + Name: "syncPullRequests", + EntryPoint: SyncPullRequests, + EnabledByDefault: true, + Description: "Sync GitHub pull requests from Snowflake into _tool_github_pull_requests", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS}, +} + +func SyncPullRequests(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + repoId := data.Options.GithubId + fullName := data.Options.Name + + var timeAfter *time.Time + syncPolicy := subtaskCtx.TaskContext().SyncPolicy() + if syncPolicy != nil { + timeAfter = syncPolicy.TimeAfter + } + + query, args := buildPullRequestsQuery(repoId, timeAfter) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for pull requests") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + githubId int64 + repositoryId int64 + headRepoId *int64 + number int64 + state string + title *string + body *string + createdAt time.Time + updatedAt time.Time + closedAt *time.Time + isDraft *bool + mergeCommitSha *string + headRef *string + baseRef *string + headSha *string + baseSha *string + authorId *int64 + authorName *string + mergedAt *time.Time + mergedById *int64 + mergedByName *string + ) + if scanErr := rows.Scan( + &githubId, &repositoryId, &headRepoId, &number, &state, &title, &body, + &createdAt, &updatedAt, &closedAt, &isDraft, &mergeCommitSha, + &headRef, &baseRef, &headSha, &baseSha, + &authorId, &authorName, &mergedAt, &mergedById, &mergedByName, + ); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan pull request row") + } + + draft := false + if isDraft != nil { + draft = *isDraft + } + merged := mergedAt != nil + + pr := &githubmodels.GithubPullRequest{ + ConnectionId: connectionId, + GithubId: int(githubId), + RepoId: int(repositoryId), + HeadRepoId: nullInt(headRepoId), + Number: int(number), + State: state, + Title: nullStr(title), + Body: nullStr(body), + GithubCreatedAt: createdAt, + GithubUpdatedAt: updatedAt, + ClosedAt: closedAt, + IsDraft: draft, + Merged: merged, + MergedAt: mergedAt, + MergeCommitSha: nullStr(mergeCommitSha), + HeadRef: nullStr(headRef), + BaseRef: nullStr(baseRef), + HeadCommitSha: nullStr(headSha), + BaseCommitSha: nullStr(baseSha), + AuthorId: nullInt(authorId), + AuthorName: nullStr(authorName), + MergedById: nullInt(mergedById), + MergedByName: nullStr(mergedByName), + Url: derivePullRequestURL(fullName, int(number)), + NoPKModel: common.NewNoPKModel(), + } + if dbErr := db.CreateOrUpdate(pr); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating pull request rows") + } + + logger.Info("synced %d pull requests from Snowflake", count) + return nil +} + +func buildPullRequestsQuery(repoId int, timeAfter *time.Time) (string, []interface{}) { + query := ` +SELECT + pr.ID AS github_id, + i.REPOSITORY_ID AS repo_id, + pr.HEAD_REPO_ID AS head_repo_id, + i.NUMBER AS number, + i.STATE AS state, + i.TITLE AS title, + i.BODY AS body, + COALESCE(pr.CREATED_AT, i.CREATED_AT) AS created_at, + pr.UPDATED_AT AS updated_at, + COALESCE(pr.CLOSED_AT, i.CLOSED_AT) AS closed_at, + pr.DRAFT AS is_draft, + pr.MERGE_COMMIT_SHA, + pr.HEAD_REF, + pr.BASE_REF, + pr.HEAD_SHA, + pr.BASE_SHA, + i.USER_ID AS author_id, + u.LOGIN AS author_name, + im.MERGED_AT, + im.ACTOR_ID AS merged_by_id, + mu.LOGIN AS merged_by_name +FROM PULL_REQUEST pr +JOIN ISSUE i + ON i.ID = pr.ISSUE_ID +LEFT JOIN "USER" u + ON u.ID = i.USER_ID +LEFT JOIN ISSUE_MERGED im + ON im.ISSUE_ID = i.ID +LEFT JOIN "USER" mu + ON mu.ID = im.ACTOR_ID +WHERE i.REPOSITORY_ID = ? +` + args := []interface{}{repoId} + if timeAfter != nil { + query += " AND pr.UPDATED_AT > ?\n" + args = append(args, *timeAfter) + } + return query, args +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go new file mode 100644 index 00000000000..4f6fc754b2c --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBuildPullRequestsQuery_NoTimeFilter(t *testing.T) { + q, args := buildPullRequestsQuery(12345, nil) + assert.Contains(t, q, "FROM PULL_REQUEST pr") + assert.Contains(t, q, "JOIN ISSUE i") + assert.Contains(t, q, "LEFT JOIN ISSUE_MERGED im") + assert.Contains(t, q, `LEFT JOIN "USER" u`) + assert.Contains(t, q, "WHERE i.REPOSITORY_ID = ?") + assert.NotContains(t, q, "pr.UPDATED_AT >") + assert.Equal(t, []interface{}{12345}, args) +} + +func TestBuildPullRequestsQuery_WithTimeFilter(t *testing.T) { + ts := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC) + q, args := buildPullRequestsQuery(12345, &ts) + assert.Contains(t, q, "AND pr.UPDATED_AT > ?") + assert.Equal(t, 2, len(args)) + assert.Equal(t, 12345, args[0]) + assert.Equal(t, ts, args[1]) +} + +func TestBuildPullRequestsQuery_RequiredColumns(t *testing.T) { + q, _ := buildPullRequestsQuery(1, nil) + for _, col := range []string{ + "pr.ID", "i.REPOSITORY_ID", "i.NUMBER", "i.STATE", "i.TITLE", + "pr.UPDATED_AT", "pr.DRAFT", "pr.MERGE_COMMIT_SHA", + "pr.HEAD_REF", "pr.BASE_REF", "i.USER_ID", "im.MERGED_AT", + } { + assert.Contains(t, q, col) + } +} diff --git a/backend/plugins/github_snowflake/tasks/sync_repos.go b/backend/plugins/github_snowflake/tasks/sync_repos.go new file mode 100644 index 00000000000..837867fc101 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_repos.go @@ -0,0 +1,109 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncReposMeta = plugin.SubTaskMeta{ + Name: "syncRepos", + EntryPoint: SyncRepos, + EnabledByDefault: true, + Description: "Sync GitHub repositories from Snowflake into _tool_github_repos", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_CODE_REVIEW, plugin.DOMAIN_TYPE_CROSS}, +} + +func SyncRepos(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + githubId := data.Options.GithubId + + query, args := buildReposQuery(githubId) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for repositories") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + id int64 + name string + fullName string + description *string + language *string + ownerId *int64 + createdAt *time.Time + ) + if scanErr := rows.Scan(&id, &name, &fullName, &description, &language, &ownerId, &createdAt); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan repository row") + } + + repo := &githubmodels.GithubRepo{ + Scope: common.Scope{ + ConnectionId: connectionId, + ScopeConfigId: data.Options.ScopeConfigId, + }, + GithubId: int(id), + Name: name, + FullName: fullName, + Description: nullStr(description), + Language: nullStr(language), + OwnerId: nullInt(ownerId), + HTMLUrl: deriveRepoHTMLUrl(fullName), + CloneUrl: deriveRepoCloneUrl(fullName), + CreatedDate: createdAt, + } + if dbErr := db.CreateOrUpdate(repo); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating repository rows") + } + + logger.Info("synced %d repositories from Snowflake", count) + return nil +} + +func buildReposQuery(githubId int) (string, []interface{}) { + query := ` +SELECT + ID, + NAME, + FULL_NAME, + DESCRIPTION, + LANGUAGE, + OWNER_ID, + CREATED_AT +FROM REPOSITORY +WHERE ID = ? +` + return query, []interface{}{githubId} +} diff --git a/backend/plugins/github_snowflake/tasks/sync_repos_test.go b/backend/plugins/github_snowflake/tasks/sync_repos_test.go new file mode 100644 index 00000000000..091fe038bf1 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_repos_test.go @@ -0,0 +1,32 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBuildReposQuery(t *testing.T) { + q, args := buildReposQuery(162) + assert.Contains(t, q, "FROM REPOSITORY") + assert.Contains(t, q, "WHERE ID = ?") + assert.Contains(t, q, "FULL_NAME") + assert.Equal(t, []interface{}{162}, args) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_reviewers.go b/backend/plugins/github_snowflake/tasks/sync_reviewers.go new file mode 100644 index 00000000000..b1d480417d1 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_reviewers.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" +) + +var SyncReviewersMeta = plugin.SubTaskMeta{ + Name: "syncReviewers", + EntryPoint: SyncReviewers, + EnabledByDefault: true, + Description: "Sync GitHub requested reviewers from Snowflake into _tool_github_reviewers", + DomainTypes: []string{plugin.DOMAIN_TYPE_CODE_REVIEW}, +} + +func SyncReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { + data := subtaskCtx.GetData().(*GithubSnowflakeTaskData) + db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() + + connectionId := data.Options.ConnectionId + repoId := data.Options.GithubId + + var timeAfter *time.Time + syncPolicy := subtaskCtx.TaskContext().SyncPolicy() + if syncPolicy != nil { + timeAfter = syncPolicy.TimeAfter + } + + query, args := buildReviewersQuery(repoId, timeAfter) + rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) + if goErr != nil { + return errors.Default.Wrap(goErr, "failed to query Snowflake for reviewers") + } + defer rows.Close() + + count := 0 + for rows.Next() { + var ( + reviewerId int64 + pullRequestId int64 + username *string + name *string + ) + if scanErr := rows.Scan(&reviewerId, &pullRequestId, &username, &name); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan reviewer row") + } + + reviewer := &githubmodels.GithubReviewer{ + ConnectionId: connectionId, + ReviewerId: int(reviewerId), + PullRequestId: int(pullRequestId), + Username: nullStr(username), + Name: nullStr(name), + NoPKModel: common.NewNoPKModel(), + } + if dbErr := db.CreateOrUpdate(reviewer); dbErr != nil { + return dbErr + } + count++ + } + if err := rows.Err(); err != nil { + return errors.Default.Wrap(err, "error iterating reviewer rows") + } + + logger.Info("synced %d reviewers from Snowflake", count) + return nil +} + +func buildReviewersQuery(repoId int, timeAfter *time.Time) (string, []interface{}) { + // Latest non-removed user request per (pull_request_id, requested_id). + query := ` +SELECT + h.REQUESTED_ID AS reviewer_id, + h.PULL_REQUEST_ID AS pull_request_id, + u.LOGIN AS username, + u.NAME AS name +FROM REQUESTED_REVIEWER_HISTORY h +JOIN PULL_REQUEST pr + ON pr.ID = h.PULL_REQUEST_ID +JOIN ISSUE i + ON i.ID = pr.ISSUE_ID +LEFT JOIN "USER" u + ON u.ID = h.REQUESTED_ID +WHERE i.REPOSITORY_ID = ? + AND LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user' + AND (h.REMOVED IS NULL OR h.REMOVED = FALSE) +` + args := []interface{}{repoId} + if timeAfter != nil { + query += " AND h.CREATED_AT > ?\n" + args = append(args, *timeAfter) + } + query += ` +QUALIFY ROW_NUMBER() OVER ( + PARTITION BY h.PULL_REQUEST_ID, h.REQUESTED_ID + ORDER BY h.CREATED_AT DESC NULLS LAST +) = 1 +` + return query, args +} diff --git a/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go b/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go new file mode 100644 index 00000000000..d677fd995ff --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestBuildReviewersQuery_FiltersUsersAndLatest(t *testing.T) { + q, args := buildReviewersQuery(42, nil) + assert.Contains(t, q, "FROM REQUESTED_REVIEWER_HISTORY h") + assert.Contains(t, q, "LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user'") + assert.Contains(t, q, "QUALIFY ROW_NUMBER()") + assert.Contains(t, q, "PARTITION BY h.PULL_REQUEST_ID, h.REQUESTED_ID") + assert.Contains(t, q, "h.REMOVED IS NULL OR h.REMOVED = FALSE") + assert.Equal(t, []interface{}{42}, args) +} + +func TestBuildReviewersQuery_WithTimeFilter(t *testing.T) { + ts := time.Date(2026, 2, 1, 0, 0, 0, 0, time.UTC) + q, args := buildReviewersQuery(42, &ts) + assert.Contains(t, q, "AND h.CREATED_AT > ?") + assert.Equal(t, ts, args[1]) +} diff --git a/backend/plugins/github_snowflake/tasks/task_data.go b/backend/plugins/github_snowflake/tasks/task_data.go new file mode 100644 index 00000000000..bbf13564d69 --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/task_data.go @@ -0,0 +1,156 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "crypto/rsa" + "crypto/x509" + "database/sql" + "encoding/pem" + "fmt" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + githubmodels "github.com/apache/incubator-devlake/plugins/github/models" + sf "github.com/snowflakedb/gosnowflake" +) + +// Raw table name constants used for StatefulDataConverter state params +// (same strings as plugins/github/tasks so state keys stay compatible). +const ( + RAW_PULL_REQUEST_TABLE = "github_api_pull_requests" + RAW_PR_COMMIT_TABLE = "github_api_pull_request_commits" + RAW_PR_REVIEW_TABLE = "github_api_pull_request_reviews" + RAW_ACCOUNT_TABLE = "github_api_accounts" +) + +// GithubSnowflakeOptions contains all per-pipeline task options. +type GithubSnowflakeOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"` + GithubId int `json:"githubId" mapstructure:"githubId"` + Name string `json:"name" mapstructure:"name"` + FullName string `json:"fullName" mapstructure:"fullName"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId"` + ScopeConfig *githubmodels.GithubScopeConfig `json:"scopeConfig" mapstructure:"scopeConfig"` +} + +// GithubSnowflakeTaskData is passed to every subtask via taskCtx.GetData(). +type GithubSnowflakeTaskData struct { + Options *GithubSnowflakeOptions + SnowflakeDB *sql.DB +} + +// GithubApiParams mirrors github/models.GithubApiParams so that RawDataSubTaskArgs +// produces the same _raw_data_params format for state management. +type GithubApiParams struct { + ConnectionId uint64 + Name string +} + +// DecodeAndValidateTaskOptions decodes and validates options for the task. +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GithubSnowflakeOptions, errors.Error) { + var op GithubSnowflakeOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New(fmt.Sprintf("invalid connectionId: %d", op.ConnectionId)) + } + if op.GithubId == 0 { + return nil, errors.BadInput.New(fmt.Sprintf("invalid githubId: %d", op.GithubId)) + } + if op.Name == "" { + op.Name = op.FullName + } + if op.FullName == "" { + op.FullName = op.Name + } + if op.Name == "" { + return nil, errors.BadInput.New("name (owner/repo full name) must not be empty") + } + return &op, nil +} + +// OpenSnowflakeDB opens a database/sql connection to Snowflake. +// +// authType controls authentication: +// - "keypair" (default): JWT key-pair auth using privateKeyPEM. Works in containers and CI. +// - "externalbrowser": SSO via browser pop-up. Only works when DevLake runs on a desktop host +// (i.e. via `make run`, not inside a Docker container). +func OpenSnowflakeDB(account, user, authType, privateKeyPEM, database, schema, warehouse, role string) (*sql.DB, errors.Error) { + cfg := &sf.Config{ + Account: account, + User: user, + Database: database, + Schema: schema, + } + if warehouse != "" { + cfg.Warehouse = warehouse + } + if role != "" { + cfg.Role = role + } + + if authType == "externalbrowser" { + cfg.Authenticator = sf.AuthTypeExternalBrowser + } else { + privKey, err := parseRSAPrivateKey(privateKeyPEM) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") + } + cfg.Authenticator = sf.AuthTypeJwt + cfg.PrivateKey = privKey + } + + dsn, goErr := sf.DSN(cfg) + if goErr != nil { + return nil, errors.Default.Wrap(goErr, "failed to build Snowflake DSN") + } + db, goErr := sql.Open("snowflake", dsn) + if goErr != nil { + return nil, errors.Default.Wrap(goErr, "failed to open Snowflake connection") + } + return db, nil +} + +// parseRSAPrivateKey parses a PKCS#8 PEM-encoded RSA private key. +func parseRSAPrivateKey(pemStr string) (*rsa.PrivateKey, error) { + block, _ := pem.Decode([]byte(pemStr)) + if block == nil { + return nil, fmt.Errorf("failed to decode PEM block from private key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse PKCS8 private key: %w", err) + } + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("private key is not an RSA key") + } + return rsaKey, nil +} + +// repoShortName returns the short name from owner/repo full name. +func repoShortName(fullName string) string { + parts := strings.Split(fullName, "/") + if len(parts) == 0 { + return fullName + } + return parts[len(parts)-1] +} diff --git a/backend/plugins/github_snowflake/tasks/task_data_test.go b/backend/plugins/github_snowflake/tasks/task_data_test.go new file mode 100644 index 00000000000..51f6ed12a4c --- /dev/null +++ b/backend/plugins/github_snowflake/tasks/task_data_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDecodeAndValidateTaskOptions_OK(t *testing.T) { + op, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + "name": "konflux-ci/build-service", + }) + assert.Nil(t, err) + assert.Equal(t, uint64(1), op.ConnectionId) + assert.Equal(t, 123, op.GithubId) + assert.Equal(t, "konflux-ci/build-service", op.Name) + assert.Equal(t, "konflux-ci/build-service", op.FullName) +} + +func TestDecodeAndValidateTaskOptions_FullNameFallback(t *testing.T) { + op, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + "fullName": "owner/repo", + }) + assert.Nil(t, err) + assert.Equal(t, "owner/repo", op.Name) +} + +func TestDecodeAndValidateTaskOptions_MissingConnection(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "githubId": 123, + "name": "owner/repo", + }) + assert.NotNil(t, err) +} + +func TestDecodeAndValidateTaskOptions_MissingGithubId(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "name": "owner/repo", + }) + assert.NotNil(t, err) +} + +func TestDecodeAndValidateTaskOptions_MissingName(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + }) + assert.NotNil(t, err) +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 84a858d4076..fe5c9a5dbb2 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -38,6 +38,7 @@ import ( gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" github "github.com/apache/incubator-devlake/plugins/github/impl" + github_snowflake "github.com/apache/incubator-devlake/plugins/github_snowflake/impl" githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" icla "github.com/apache/incubator-devlake/plugins/icla/impl" @@ -83,6 +84,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("gitee/models", gitee.Gitee{}.GetTablesInfo) checker.FeedIn("gitextractor/models", gitextractor.GitExtractor{}.GetTablesInfo) checker.FeedIn("github/models", github.Github{}.GetTablesInfo) + checker.FeedIn("github_snowflake/models", github_snowflake.GithubSnowflake{}.GetTablesInfo) checker.FeedIn("github_graphql", githubGraphql.GithubGraphql{}.GetTablesInfo) checker.FeedIn("gitlab/models", gitlab.Gitlab{}.GetTablesInfo) checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) diff --git a/docs/github-snowflake-local-testing.md b/docs/github-snowflake-local-testing.md new file mode 100644 index 00000000000..36d7e0876d2 --- /dev/null +++ b/docs/github-snowflake-local-testing.md @@ -0,0 +1,263 @@ +# github_snowflake Plugin — Local Testing Guide + +## Prerequisites + +- Go 1.21+ +- podman + podman-compose +- Access to the Snowflake account (`GITHUB_DB.MARTS`) +- Snowflake role with `SELECT` on that schema (e.g. `GITHUB_GROUP`) +- A konflux-ci repo to pilot (`githubId` + `owner/repo` full name) + +--- + +## Step 1 — Verify Snowflake access + +Before starting DevLake, confirm you can reach Snowflake from your machine. + +Create a throwaway Go file (outside the repo): + +```go +// /tmp/check_github_snowflake.go +package main + +import ( + "database/sql" + "fmt" + "log" + + sf "github.com/snowflakedb/gosnowflake" +) + +func main() { + cfg := &sf.Config{ + Account: "YOUR_ACCOUNT", // e.g. "myorg-myaccount" (not the full URL) + User: "YOUR_USER", + Role: "GITHUB_GROUP", + Warehouse: "DEFAULT", + Database: "GITHUB_DB", + Schema: "MARTS", + Authenticator: sf.AuthTypeExternalBrowser, // opens browser for SSO + } + dsn, err := sf.DSN(cfg) + if err != nil { + log.Fatal(err) + } + + db, err := sql.Open("snowflake", dsn) + if err != nil { + log.Fatal(err) + } + defer db.Close() + + var n int + row := db.QueryRow("SELECT COUNT(*) FROM REPOSITORY") + if err := row.Scan(&n); err != nil { + log.Fatal(err) + } + fmt.Printf("OK — %d repositories visible\n", n) +} +``` + +Run it from `backend/` where `gosnowflake` is already in `go.mod`: + +```bash +cd backend +go run /tmp/check_github_snowflake.go +# A browser window opens for SSO — log in once, then the script prints: +# OK — NNN repositories visible +``` + +Pick a pilot repo ID and full name: + +```sql +SELECT ID, FULL_NAME FROM REPOSITORY ORDER BY FULL_NAME LIMIT 20; +``` + +**Common errors:** + +| Error | Fix | +|---|---| +| `account is empty` | Use the account identifier (e.g. `myorg-myaccount`), not the full `*.snowflakecomputing.com` URL | +| `Object does not exist or not authorized` | Your role lacks `SELECT` on `GITHUB_DB.MARTS` — ask a Snowflake admin | +| Browser window never opens | You may be running inside a container or headless SSH session — run locally | + +--- + +## Step 2 — Start MySQL + +DevLake needs MySQL for connections, tool-layer tables, and domain tables. Unit tests do **not** need MySQL; a real pipeline run does. + +```bash +podman machine start # if not already running +podman compose -f docker-compose-dev.yml up -d mysql +``` + +Confirm MySQL is ready: + +```bash +podman compose -f docker-compose-dev.yml exec mysql \ + mysql -umerico -pmerico lake -e "SELECT 1;" +``` + +--- + +## Step 3 — Configure `.env` + +Create or edit `.env` in the repo root: + +```env +# Disable auth for local testing (skip bearer token on every API call) +AUTH_ENABLED=false + +# Required: any 32+ char string works for local dev — replace with your own value +ENCRYPTION_SECRET= + +DB_URL=mysql://merico:merico@127.0.0.1:3306/lake?charset=utf8mb4&parseTime=True + +# Generate ENCRYPTION_SECRET with: +# openssl rand -hex 16 +``` + +--- + +## Step 4 — Build and run DevLake (plugin only) + +Browser-based SSO (`externalbrowser` auth) only works when DevLake runs **natively on your desktop**, not inside a container (the browser pop-up cannot reach a container process). + +```bash +cd backend +DEVLAKE_PLUGINS=github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run +``` + +- `DEVLAKE_PLUGINS=github_snowflake` — only compile this plugin (much faster than building all plugins) +- `DISABLED_REMOTE_PLUGINS=true` — skip loading remote/dynamic plugins +- `ENV_FILE=../.env` — point to the `.env` in the repo root + +Verify the plugin loaded: + +```bash +curl -s http://localhost:8080/plugins | jq '.[] | select(.plugin == "github_snowflake")' +``` + +Trigger DB migrations (creates `_tool_github_snowflake_connections` table): + +```bash +curl -s http://localhost:8080/proceed-db-migration | jq . +``` + +Verify the table exists: + +```bash +podman compose -f docker-compose-dev.yml exec mysql \ + mysql -umerico -pmerico lake -e "SHOW TABLES LIKE '_tool_github_snowflake%';" +``` + +--- + +## Step 5 — Create a connection + +```bash +curl -s -X POST http://localhost:8080/plugins/github_snowflake/connections \ + -H 'Content-Type: application/json' \ + -d '{ + "name": "my-snowflake-github", + "account": "YOUR_ACCOUNT", + "user": "YOUR_USER", + "authType": "externalbrowser", + "database": "GITHUB_DB", + "schema": "MARTS", + "warehouse": "DEFAULT", + "role": "GITHUB_GROUP" + }' | jq . +``` + +Note the `id` in the response — that is your `connectionId`. + +> For **key-pair auth** (production / CI), use `"authType": "keypair"` and +> add `"privateKey": "-----BEGIN PRIVATE KEY-----\n..."`. +> Key-pair auth works inside containers and does not require a browser. + +--- + +## Step 6 — Run a pipeline + +You need: +- `connectionId` from Step 5 +- `githubId`: numeric GitHub repo ID from Snowflake `REPOSITORY.ID` +- `name` / `fullName`: `owner/repo` (e.g. `konflux-ci/build-service`) + +```bash +curl -s -X POST http://localhost:8080/pipelines \ + -H 'Content-Type: application/json' \ + -d '{ + "name": "github-snowflake-test", + "plan": [[{ + "plugin": "github_snowflake", + "options": { + "connectionId": 1, + "githubId": 123456789, + "name": "konflux-ci/build-service", + "fullName": "konflux-ci/build-service" + } + }]] + }' | jq '{id, status}' +``` + +Poll the pipeline until it completes: + +```bash +PIPELINE_ID= +while true; do + curl -s http://localhost:8080/pipelines/$PIPELINE_ID | jq '{status, message}' + sleep 5 +done +``` + +A successful run ends with `"status": "TASK_COMPLETED"`. + +On the first `externalbrowser` sync, Snowflake opens a browser window for SSO — complete login once. + +--- + +## Step 7 — Verify results in MySQL + +```bash +MYSQL="podman compose -f docker-compose-dev.yml exec mysql mysql -umerico -pmerico lake -e" + +# Row counts — tool layer and domain layer +$MYSQL "SELECT + (SELECT COUNT(*) FROM _tool_github_repos WHERE connection_id = 1) AS tool_repos, + (SELECT COUNT(*) FROM _tool_github_pull_requests WHERE connection_id = 1) AS tool_prs, + (SELECT COUNT(*) FROM pull_requests) AS domain_prs, + (SELECT COUNT(*) FROM _tool_github_pull_request_reviews WHERE connection_id = 1) AS tool_reviews, + (SELECT COUNT(*) FROM _tool_github_accounts WHERE connection_id = 1) AS tool_accounts\G" + +# Sample PRs +$MYSQL "SELECT number, state, title, author_name, merged + FROM _tool_github_pull_requests + WHERE connection_id = 1 + ORDER BY github_updated_at DESC + LIMIT 10;" +``` + +**Expected results:** +- At least one row in `_tool_github_repos` for the pilot repo +- `tool_prs` > 0 for an active konflux-ci repo; `domain_prs` should match after convertors run +- Author/reviewer logins populated (unlike Jira Snowflake PII stripping) + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| Plugin not listed at `/plugins` | Build step skipped or failed | Re-run `make build-plugin run` and check for compile errors | +| `ENCRYPTION_SECRET` error on startup | `.env` missing or key too short | Set a 32+ char hex string in `.env` | +| `401 Unauthorized` on API calls | Auth enabled | Set `AUTH_ENABLED=false` in `.env` | +| MySQL connection refused | MySQL not running | `podman compose -f docker-compose-dev.yml up -d mysql` | +| `invalid identifier 'X'` in pipeline logs | Column/table name mismatch | `DESCRIBE TABLE GITHUB_DB.MARTS.;` in Snowflake | +| `Object 'USER' does not exist` | Unquoted reserved keyword | Queries must use `"USER"` (already done in sync tasks) | +| Browser pop-up doesn't open | Running inside a container | Run DevLake natively with `make run`, not via `podman compose` | +| Pipeline ends with `TASK_FAILED` | Check `message` field | `curl -s http://localhost:8080/pipelines/$ID \| jq .message` | +| `tool_prs` populated but `domain_prs` = 0 | Convertor subtask failed | Check pipeline subtask logs; ensure convertors are enabled | +| Same repo also on GitHub API connection | Domain ID duplication risk | Remove the repo from one of the two connections | diff --git a/docs/upstream-diffs.md b/docs/upstream-diffs.md index 5fc62b514c2..f7ac4d772d1 100644 --- a/docs/upstream-diffs.md +++ b/docs/upstream-diffs.md @@ -3,12 +3,15 @@ This file tracks modifications to files originating from [apache/incubator-devlake](https://github.com/apache/incubator-devlake) that must be maintained during upstream syncs. -Owned plugins (`aireview`, `codecov`, `testregistry`, `agentready`, `langfuse`, `jira_snowflake`) are additions, +Owned plugins (`aireview`, `codecov`, `testregistry`, `agentready`, `langfuse`, `jira_snowflake`, `github_snowflake`) are additions, not modifications, and are not tracked here. `jira_snowflake/tasks/convert_*.go` are adapted copies of `jira/tasks/` convertors — see the [jira_snowflake AGENTS.md](../backend/plugins/jira_snowflake/AGENTS.md) for the diff details. +`github_snowflake/tasks/convert_*.go` are adapted copies of `github/tasks/` convertors — see the +[github_snowflake AGENTS.md](../backend/plugins/github_snowflake/AGENTS.md) for the diff details. + ## gitextractor: ForceFullClone / FORCE_FULL_GIT_HISTORY **Files:** From dc665603836b7ee2ea55e6515d56599d11057dcc Mon Sep 17 00:00:00 2001 From: fmuntean Date: Mon, 3 Aug 2026 12:32:44 +0200 Subject: [PATCH 02/10] fix(github_snowflake): address Full Send review findings Encrypt Snowflake private keys at rest via gorm encdec (including jira_snowflake), handle nullable PR updated_at, and tighten AuthType and owner/repo validation. Co-Authored-By: Cursor Grok 4.5 Co-authored-by: Cursor --- backend/plugins/github_snowflake/impl/impl.go | 11 +-- .../github_snowflake/models/connection.go | 4 +- .../migrationscripts/encrypt_private_key.go | 83 +++++++++++++++++++ .../models/migrationscripts/init_schema.go | 2 +- .../models/migrationscripts/register.go | 1 + .../github_snowflake/tasks/shared_test.go | 4 +- .../tasks/sync_pull_requests.go | 8 +- .../github_snowflake/tasks/task_data.go | 30 +++++-- .../github_snowflake/tasks/task_data_test.go | 29 +++++++ .../jira_snowflake/models/connection.go | 4 +- .../migrationscripts/encrypt_private_key.go | 83 +++++++++++++++++++ .../models/migrationscripts/register.go | 1 + .../plugins/jira_snowflake/tasks/task_data.go | 11 ++- backend/plugins/table_info_test.go | 6 +- 14 files changed, 247 insertions(+), 30 deletions(-) create mode 100644 backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go create mode 100644 backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go diff --git a/backend/plugins/github_snowflake/impl/impl.go b/backend/plugins/github_snowflake/impl/impl.go index 7d74be327c7..d283838d620 100644 --- a/backend/plugins/github_snowflake/impl/impl.go +++ b/backend/plugins/github_snowflake/impl/impl.go @@ -124,19 +124,10 @@ func (p GithubSnowflake) PrepareTaskData(taskCtx plugin.TaskContext, options map repo := &githubmodels.GithubRepo{} dbErr := taskCtx.GetDal().First(repo, dal.Where("connection_id = ? AND github_id = ?", op.ConnectionId, op.GithubId)) if dbErr != nil && taskCtx.GetDal().IsErrorNotFound(dbErr) { - shortName := op.Name - if idx := len(op.Name) - 1; idx >= 0 { - for i := len(op.Name) - 1; i >= 0; i-- { - if op.Name[i] == '/' { - shortName = op.Name[i+1:] - break - } - } - } repo = &githubmodels.GithubRepo{ Scope: coremodels.Scope{ConnectionId: op.ConnectionId}, GithubId: op.GithubId, - Name: shortName, + Name: tasks.RepoShortName(op.Name), FullName: op.Name, HTMLUrl: "https://github.com/" + op.Name, CloneUrl: "https://github.com/" + op.Name + ".git", diff --git a/backend/plugins/github_snowflake/models/connection.go b/backend/plugins/github_snowflake/models/connection.go index 3770d8fa3b9..5886c712b9a 100644 --- a/backend/plugins/github_snowflake/models/connection.go +++ b/backend/plugins/github_snowflake/models/connection.go @@ -22,7 +22,7 @@ import ( ) // SnowflakeGithubConnection holds the credentials and config for a Snowflake-backed GitHub connection. -// The private key PEM is stored encrypted (encrypt:"yes") using DevLake's field-level encryption. +// The private key PEM is stored encrypted via GORM's encdec serializer. type SnowflakeGithubConnection struct { helper.BaseConnection `mapstructure:",squash"` // Account is the Snowflake account identifier, e.g. "myorg-myaccount" @@ -35,7 +35,7 @@ type SnowflakeGithubConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - PrivateKey string `json:"privateKey" encrypt:"yes" gorm:"column:private_key" mapstructure:"privateKey"` + PrivateKey string `json:"privateKey" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "GITHUB_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` // Schema is the Snowflake schema, e.g. "MARTS" diff --git a/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go new file mode 100644 index 00000000000..86bf4dcf0a1 --- /dev/null +++ b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +type githubSnowflakePrivateKeyPlain struct { + ID uint64 `gorm:"primaryKey"` + PrivateKey string `gorm:"column:private_key"` +} + +func (githubSnowflakePrivateKeyPlain) TableName() string { + return "_tool_github_snowflake_connections" +} + +// encryptPrivateKey encrypts existing plaintext private_key values so that the +// connection model's gorm serializer:encdec can decrypt them on read. +type encryptPrivateKey struct{} + +func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + encKey := basicRes.GetConfig(plugin.EncodeKeyEnvStr) + if encKey == "" { + return errors.BadInput.New("ENCRYPTION_SECRET is required to encrypt github_snowflake private keys") + } + + cursor, err := db.Cursor(dal.From(&githubSnowflakePrivateKeyPlain{})) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + row := &githubSnowflakePrivateKeyPlain{} + if err = db.Fetch(cursor, row); err != nil { + return err + } + if row.PrivateKey == "" { + continue + } + encrypted, err := plugin.Encrypt(encKey, row.PrivateKey) + if err != nil { + return err + } + err = db.UpdateColumns( + row.TableName(), + []dal.DalSet{{ColumnName: "private_key", Value: encrypted}}, + dal.Where("id = ?", row.ID), + ) + if err != nil { + return err + } + } + return nil +} + +func (*encryptPrivateKey) Version() uint64 { + return 20260803000001 +} + +func (*encryptPrivateKey) Name() string { + return "encrypt github_snowflake connection private keys" +} diff --git a/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go b/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go index b7d6c94a689..58e724fab5c 100644 --- a/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go +++ b/backend/plugins/github_snowflake/models/migrationscripts/init_schema.go @@ -32,7 +32,7 @@ type snowflakeGithubConnection20260731000001 struct { Account string `gorm:"column:account;not null"` User string `gorm:"column:sf_user;not null"` AuthType string `gorm:"column:auth_type;default:keypair"` - PrivateKey string `gorm:"column:private_key"` + PrivateKey string `gorm:"column:private_key;type:text;serializer:encdec"` Database string `gorm:"column:sf_database;not null"` Schema string `gorm:"column:sf_schema;not null"` Warehouse string `gorm:"column:warehouse"` diff --git a/backend/plugins/github_snowflake/models/migrationscripts/register.go b/backend/plugins/github_snowflake/models/migrationscripts/register.go index c0cd8affcfc..48e3886013b 100644 --- a/backend/plugins/github_snowflake/models/migrationscripts/register.go +++ b/backend/plugins/github_snowflake/models/migrationscripts/register.go @@ -23,5 +23,6 @@ import "github.com/apache/incubator-devlake/core/plugin" func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(initSchema), + new(encryptPrivateKey), } } diff --git a/backend/plugins/github_snowflake/tasks/shared_test.go b/backend/plugins/github_snowflake/tasks/shared_test.go index 54849817e93..9b84bb7da9c 100644 --- a/backend/plugins/github_snowflake/tasks/shared_test.go +++ b/backend/plugins/github_snowflake/tasks/shared_test.go @@ -48,6 +48,6 @@ func TestNullInt(t *testing.T) { } func TestRepoShortName(t *testing.T) { - assert.Equal(t, "build-service", repoShortName("konflux-ci/build-service")) - assert.Equal(t, "alone", repoShortName("alone")) + assert.Equal(t, "build-service", RepoShortName("konflux-ci/build-service")) + assert.Equal(t, "alone", RepoShortName("alone")) } diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go index 1eac8838436..09882601ea4 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go @@ -67,7 +67,7 @@ func SyncPullRequests(subtaskCtx plugin.SubTaskContext) errors.Error { title *string body *string createdAt time.Time - updatedAt time.Time + updatedAt *time.Time closedAt *time.Time isDraft *bool mergeCommitSha *string @@ -95,6 +95,10 @@ func SyncPullRequests(subtaskCtx plugin.SubTaskContext) errors.Error { draft = *isDraft } merged := mergedAt != nil + prUpdatedAt := createdAt + if updatedAt != nil { + prUpdatedAt = *updatedAt + } pr := &githubmodels.GithubPullRequest{ ConnectionId: connectionId, @@ -106,7 +110,7 @@ func SyncPullRequests(subtaskCtx plugin.SubTaskContext) errors.Error { Title: nullStr(title), Body: nullStr(body), GithubCreatedAt: createdAt, - GithubUpdatedAt: updatedAt, + GithubUpdatedAt: prUpdatedAt, ClosedAt: closedAt, IsDraft: draft, Merged: merged, diff --git a/backend/plugins/github_snowflake/tasks/task_data.go b/backend/plugins/github_snowflake/tasks/task_data.go index bbf13564d69..95e9d491147 100644 --- a/backend/plugins/github_snowflake/tasks/task_data.go +++ b/backend/plugins/github_snowflake/tasks/task_data.go @@ -84,9 +84,24 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*GithubSnowfl if op.Name == "" { return nil, errors.BadInput.New("name (owner/repo full name) must not be empty") } + if err := validateOwnerRepo(op.Name); err != nil { + return nil, err + } return &op, nil } +// validateOwnerRepo checks that name is in "owner/repo" format. +func validateOwnerRepo(name string) errors.Error { + if strings.Count(name, "/") != 1 { + return errors.BadInput.New("name must be in owner/repo format") + } + owner, repo, _ := strings.Cut(name, "/") + if owner == "" || repo == "" { + return errors.BadInput.New("name must be in owner/repo format") + } + return nil +} + // OpenSnowflakeDB opens a database/sql connection to Snowflake. // // authType controls authentication: @@ -107,15 +122,20 @@ func OpenSnowflakeDB(account, user, authType, privateKeyPEM, database, schema, w cfg.Role = role } - if authType == "externalbrowser" { - cfg.Authenticator = sf.AuthTypeExternalBrowser - } else { + switch authType { + case "", "keypair": privKey, err := parseRSAPrivateKey(privateKeyPEM) if err != nil { return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") } cfg.Authenticator = sf.AuthTypeJwt cfg.PrivateKey = privKey + case "externalbrowser": + cfg.Authenticator = sf.AuthTypeExternalBrowser + default: + return nil, errors.BadInput.New(fmt.Sprintf( + `unsupported authType %q; must be "keypair" or "externalbrowser"`, authType, + )) } dsn, goErr := sf.DSN(cfg) @@ -146,8 +166,8 @@ func parseRSAPrivateKey(pemStr string) (*rsa.PrivateKey, error) { return rsaKey, nil } -// repoShortName returns the short name from owner/repo full name. -func repoShortName(fullName string) string { +// RepoShortName returns the short name from owner/repo full name. +func RepoShortName(fullName string) string { parts := strings.Split(fullName, "/") if len(parts) == 0 { return fullName diff --git a/backend/plugins/github_snowflake/tasks/task_data_test.go b/backend/plugins/github_snowflake/tasks/task_data_test.go index 51f6ed12a4c..572e5110794 100644 --- a/backend/plugins/github_snowflake/tasks/task_data_test.go +++ b/backend/plugins/github_snowflake/tasks/task_data_test.go @@ -69,3 +69,32 @@ func TestDecodeAndValidateTaskOptions_MissingName(t *testing.T) { }) assert.NotNil(t, err) } + +func TestDecodeAndValidateTaskOptions_InvalidNameFormat(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + "name": "not-a-repo", + }) + assert.NotNil(t, err) + + _, err = DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + "name": "/repo", + }) + assert.NotNil(t, err) + + _, err = DecodeAndValidateTaskOptions(map[string]interface{}{ + "connectionId": uint64(1), + "githubId": 123, + "name": "owner/", + }) + assert.NotNil(t, err) +} + +func TestOpenSnowflakeDB_InvalidAuthType(t *testing.T) { + _, err := OpenSnowflakeDB("acct", "user", "keypai", "", "db", "schema", "", "") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "unsupported authType") +} diff --git a/backend/plugins/jira_snowflake/models/connection.go b/backend/plugins/jira_snowflake/models/connection.go index cf8580ab47d..5d96993ae79 100644 --- a/backend/plugins/jira_snowflake/models/connection.go +++ b/backend/plugins/jira_snowflake/models/connection.go @@ -22,7 +22,7 @@ import ( ) // SnowflakeJiraConnection holds the credentials and config for a Snowflake-backed Jira connection. -// The private key PEM is stored encrypted (encrypt:"yes") using DevLake's field-level encryption. +// The private key PEM is stored encrypted via GORM's encdec serializer. type SnowflakeJiraConnection struct { helper.BaseConnection `mapstructure:",squash"` // Account is the Snowflake account identifier, e.g. "myorg-myaccount" @@ -35,7 +35,7 @@ type SnowflakeJiraConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - PrivateKey string `json:"privateKey" encrypt:"yes" gorm:"column:private_key" mapstructure:"privateKey"` + PrivateKey string `json:"privateKey" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "JIRA_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` // Schema is the Snowflake schema, e.g. "CLOUDRHAI_MARTS" diff --git a/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go new file mode 100644 index 00000000000..7e1e9d39e66 --- /dev/null +++ b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +type jiraSnowflakePrivateKeyPlain struct { + ID uint64 `gorm:"primaryKey"` + PrivateKey string `gorm:"column:private_key"` +} + +func (jiraSnowflakePrivateKeyPlain) TableName() string { + return "_tool_jira_snowflake_connections" +} + +// encryptPrivateKey encrypts existing plaintext private_key values so that the +// connection model's gorm serializer:encdec can decrypt them on read. +type encryptPrivateKey struct{} + +func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + encKey := basicRes.GetConfig(plugin.EncodeKeyEnvStr) + if encKey == "" { + return errors.BadInput.New("ENCRYPTION_SECRET is required to encrypt jira_snowflake private keys") + } + + cursor, err := db.Cursor(dal.From(&jiraSnowflakePrivateKeyPlain{})) + if err != nil { + return err + } + defer cursor.Close() + + for cursor.Next() { + row := &jiraSnowflakePrivateKeyPlain{} + if err = db.Fetch(cursor, row); err != nil { + return err + } + if row.PrivateKey == "" { + continue + } + encrypted, err := plugin.Encrypt(encKey, row.PrivateKey) + if err != nil { + return err + } + err = db.UpdateColumns( + row.TableName(), + []dal.DalSet{{ColumnName: "private_key", Value: encrypted}}, + dal.Where("id = ?", row.ID), + ) + if err != nil { + return err + } + } + return nil +} + +func (*encryptPrivateKey) Version() uint64 { + return 20260803000001 +} + +func (*encryptPrivateKey) Name() string { + return "encrypt jira_snowflake connection private keys" +} diff --git a/backend/plugins/jira_snowflake/models/migrationscripts/register.go b/backend/plugins/jira_snowflake/models/migrationscripts/register.go index 62b9b6447ce..654860f7478 100644 --- a/backend/plugins/jira_snowflake/models/migrationscripts/register.go +++ b/backend/plugins/jira_snowflake/models/migrationscripts/register.go @@ -24,5 +24,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(initSchema), new(addAuthType), + new(encryptPrivateKey), } } diff --git a/backend/plugins/jira_snowflake/tasks/task_data.go b/backend/plugins/jira_snowflake/tasks/task_data.go index ea1958fb203..831f40ac81f 100644 --- a/backend/plugins/jira_snowflake/tasks/task_data.go +++ b/backend/plugins/jira_snowflake/tasks/task_data.go @@ -92,15 +92,20 @@ func OpenSnowflakeDB(account, user, authType, privateKeyPEM, database, schema, w cfg.Role = role } - if authType == "externalbrowser" { - cfg.Authenticator = sf.AuthTypeExternalBrowser - } else { + switch authType { + case "", "keypair": privKey, err := parseRSAPrivateKey(privateKeyPEM) if err != nil { return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") } cfg.Authenticator = sf.AuthTypeJwt cfg.PrivateKey = privKey + case "externalbrowser": + cfg.Authenticator = sf.AuthTypeExternalBrowser + default: + return nil, errors.BadInput.New(fmt.Sprintf( + `unsupported authType %q; must be "keypair" or "externalbrowser"`, authType, + )) } dsn, goErr := sf.DSN(cfg) diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index fe5c9a5dbb2..fcde9882c06 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -28,8 +28,8 @@ import ( bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" - claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" @@ -38,8 +38,8 @@ import ( gitee "github.com/apache/incubator-devlake/plugins/gitee/impl" gitextractor "github.com/apache/incubator-devlake/plugins/gitextractor/impl" github "github.com/apache/incubator-devlake/plugins/github/impl" - github_snowflake "github.com/apache/incubator-devlake/plugins/github_snowflake/impl" githubGraphql "github.com/apache/incubator-devlake/plugins/github_graphql/impl" + github_snowflake "github.com/apache/incubator-devlake/plugins/github_snowflake/impl" gitlab "github.com/apache/incubator-devlake/plugins/gitlab/impl" icla "github.com/apache/incubator-devlake/plugins/icla/impl" issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" @@ -84,8 +84,8 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("gitee/models", gitee.Gitee{}.GetTablesInfo) checker.FeedIn("gitextractor/models", gitextractor.GitExtractor{}.GetTablesInfo) checker.FeedIn("github/models", github.Github{}.GetTablesInfo) - checker.FeedIn("github_snowflake/models", github_snowflake.GithubSnowflake{}.GetTablesInfo) checker.FeedIn("github_graphql", githubGraphql.GithubGraphql{}.GetTablesInfo) + checker.FeedIn("github_snowflake/models", github_snowflake.GithubSnowflake{}.GetTablesInfo) checker.FeedIn("gitlab/models", gitlab.Gitlab{}.GetTablesInfo) checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) checker.FeedIn("jenkins/models", jenkins.Jenkins{}.GetTablesInfo) From 7b025bf1cb1cfd04644ca136f0faa417ef2cf927 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Mon, 3 Aug 2026 12:38:37 +0200 Subject: [PATCH 03/10] fix(github_snowflake): pass connectionId in CLI DirectRun Require connectionId for standalone runs (and jira_snowflake) and note that native make run needs localhost in DB_URL. Co-Authored-By: Cursor Grok 4.5 Co-authored-by: Cursor --- backend/plugins/github_snowflake/AGENTS.md | 4 +++- backend/plugins/github_snowflake/github_snowflake.go | 11 ++++++++--- backend/plugins/jira_snowflake/jira_snowflake.go | 9 +++++++-- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/backend/plugins/github_snowflake/AGENTS.md b/backend/plugins/github_snowflake/AGENTS.md index 14231a64abf..aaa4e48461a 100644 --- a/backend/plugins/github_snowflake/AGENTS.md +++ b/backend/plugins/github_snowflake/AGENTS.md @@ -19,7 +19,9 @@ Unit tests do **not** need MySQL. A real pipeline run does. ### Minimal real-run setup 1. Start MySQL: `podman compose -f docker-compose-dev.yml up -d mysql` -2. Ensure `.env` has `DB_URL`, `ENCRYPTION_SECRET`, and (for local API) `AUTH_ENABLED=false` +2. Ensure `.env` has `DB_URL`, `ENCRYPTION_SECRET`, and (for local API) `AUTH_ENABLED=false`. + For native `make run`, use `localhost` (or `127.0.0.1`) in `DB_URL` — `env.example`'s + `@mysql:3306` host only resolves inside the compose network. 3. Run the server (native desktop required for `externalbrowser` SSO): ```bash diff --git a/backend/plugins/github_snowflake/github_snowflake.go b/backend/plugins/github_snowflake/github_snowflake.go index e9c3502b819..125da594c2a 100644 --- a/backend/plugins/github_snowflake/github_snowflake.go +++ b/backend/plugins/github_snowflake/github_snowflake.go @@ -28,15 +28,20 @@ var PluginEntry impl.GithubSnowflake //nolint func main() { cmd := &cobra.Command{Use: "github_snowflake"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "github_snowflake connection id") githubId := cmd.Flags().IntP("githubId", "g", 0, "GitHub repository numeric ID") fullName := cmd.Flags().StringP("fullName", "n", "", "GitHub repository full name (owner/repo)") timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "only sync records created/updated after this time (RFC3339)") + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("githubId") + _ = cmd.MarkFlagRequired("fullName") cmd.Run = func(cmd *cobra.Command, args []string) { runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - "githubId": *githubId, - "name": *fullName, - "fullName": *fullName, + "connectionId": *connectionId, + "githubId": *githubId, + "name": *fullName, + "fullName": *fullName, }, *timeAfter) } runner.RunCmd(cmd) diff --git a/backend/plugins/jira_snowflake/jira_snowflake.go b/backend/plugins/jira_snowflake/jira_snowflake.go index 3a137297eb7..5fef9cbbdee 100644 --- a/backend/plugins/jira_snowflake/jira_snowflake.go +++ b/backend/plugins/jira_snowflake/jira_snowflake.go @@ -28,14 +28,19 @@ var PluginEntry impl.JiraSnowflake //nolint func main() { cmd := &cobra.Command{Use: "jira_snowflake"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "jira_snowflake connection id") boardId := cmd.Flags().Uint64P("boardId", "b", 0, "Jira board ID to sync") projectKeys := cmd.Flags().StringSliceP("projectKeys", "p", nil, "Jira project keys for this board") timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "only sync records created/updated after this time (RFC3339)") + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("boardId") + _ = cmd.MarkFlagRequired("projectKeys") cmd.Run = func(cmd *cobra.Command, args []string) { runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ - "boardId": *boardId, - "projectKeys": *projectKeys, + "connectionId": *connectionId, + "boardId": *boardId, + "projectKeys": *projectKeys, }, *timeAfter) } runner.RunCmd(cmd) From 171f31b81e04bcd8cca00292e6d9ca9d0cbb2638 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Mon, 3 Aug 2026 14:14:58 +0200 Subject: [PATCH 04/10] docs(github_snowflake): require github plugin for fresh-DB local setup Document that github migrations must create shared _tool_github_* tables before pipelines run, and add troubleshooting for the common failures. Co-Authored-By: Cursor Grok 4.5 Co-authored-by: Cursor --- backend/plugins/github_snowflake/AGENTS.md | 10 +++++---- docs/github-snowflake-local-testing.md | 26 +++++++++++++++------- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/backend/plugins/github_snowflake/AGENTS.md b/backend/plugins/github_snowflake/AGENTS.md index aaa4e48461a..a83bf36a4c4 100644 --- a/backend/plugins/github_snowflake/AGENTS.md +++ b/backend/plugins/github_snowflake/AGENTS.md @@ -26,10 +26,11 @@ Unit tests do **not** need MySQL. A real pipeline run does. ```bash cd backend -DEVLAKE_PLUGINS=github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run +# Include github so its migrations create shared _tool_github_* tables (needed on a fresh DB). +DEVLAKE_PLUGINS=github,github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run ``` -4. Create a connection + run a pipeline for one repo (`githubId` + `fullName`) +4. Create a connection (`authType: externalbrowser` for local desktop) + run a pipeline for one repo (`githubId` + `fullName`) Full step-by-step (Snowflake check, migrations, curl examples, result queries): [docs/github-snowflake-local-testing.md](../../../docs/github-snowflake-local-testing.md) @@ -84,8 +85,9 @@ Pilot coverage today: **konflux-ci** org only in MARTS. ## GitHub plugin models dependency Imports `plugins/github/models` for tool-layer structs. This is a shared schema -dependency, not a business-logic cross-import. The github plugin does not need to -be deployed alongside github_snowflake — `impl.Init` registers a minimal +dependency, not a business-logic cross-import. Runtime API/business logic does +not require the github plugin, but a fresh DB still needs github loaded once so +its migrations create `_tool_github_*`. `impl.Init` registers a minimal `githubPluginStub` for didgen. ## Don'ts diff --git a/docs/github-snowflake-local-testing.md b/docs/github-snowflake-local-testing.md index 36d7e0876d2..a9b9c41163e 100644 --- a/docs/github-snowflake-local-testing.md +++ b/docs/github-snowflake-local-testing.md @@ -120,36 +120,43 @@ DB_URL=mysql://merico:merico@127.0.0.1:3306/lake?charset=utf8mb4&parseTime=True --- -## Step 4 — Build and run DevLake (plugin only) +## Step 4 — Build and run DevLake Browser-based SSO (`externalbrowser` auth) only works when DevLake runs **natively on your desktop**, not inside a container (the browser pop-up cannot reach a container process). +`github_snowflake` writes into the shared `_tool_github_*` tables owned by the **github** plugin. On a fresh database those tables only exist after github's migrations run, so include `github` in `DEVLAKE_PLUGINS` at least for the first boot: + ```bash cd backend -DEVLAKE_PLUGINS=github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run +DEVLAKE_PLUGINS=github,github_snowflake DISABLED_REMOTE_PLUGINS=true ENV_FILE=../.env make build-plugin run ``` -- `DEVLAKE_PLUGINS=github_snowflake` — only compile this plugin (much faster than building all plugins) +- `DEVLAKE_PLUGINS=github,github_snowflake` — load github (for `_tool_github_*` schema) + this plugin - `DISABLED_REMOTE_PLUGINS=true` — skip loading remote/dynamic plugins - `ENV_FILE=../.env` — point to the `.env` in the repo root -Verify the plugin loaded: +After the tool tables exist, you can drop back to `DEVLAKE_PLUGINS=github_snowflake` for faster rebuilds if you prefer. + +Verify the plugins loaded: ```bash -curl -s http://localhost:8080/plugins | jq '.[] | select(.plugin == "github_snowflake")' +curl -s http://localhost:8080/plugins | jq '.[] | select(.plugin == "github_snowflake" or .plugin == "github")' ``` -Trigger DB migrations (creates `_tool_github_snowflake_connections` table): +Trigger DB migrations (creates `_tool_github_snowflake_connections` and, via github, `_tool_github_*`): ```bash curl -s http://localhost:8080/proceed-db-migration | jq . ``` -Verify the table exists: +Verify the tables exist: ```bash podman compose -f docker-compose-dev.yml exec mysql \ - mysql -umerico -pmerico lake -e "SHOW TABLES LIKE '_tool_github_snowflake%';" + mysql -umerico -pmerico lake -e " + SHOW TABLES LIKE '_tool_github_snowflake%'; + SHOW TABLES LIKE '_tool_github_repos'; + " ``` --- @@ -259,5 +266,8 @@ $MYSQL "SELECT number, state, title, author_name, merged | `Object 'USER' does not exist` | Unquoted reserved keyword | Queries must use `"USER"` (already done in sync tasks) | | Browser pop-up doesn't open | Running inside a container | Run DevLake natively with `make run`, not via `podman compose` | | Pipeline ends with `TASK_FAILED` | Check `message` field | `curl -s http://localhost:8080/pipelines/$ID \| jq .message` | +| `Table 'lake._tool_github_repos' doesn't exist` | github plugin migrations never ran | Restart with `DEVLAKE_PLUGINS=github,github_snowflake` and re-run `/proceed-db-migration` | +| `failed to decode PEM block from private key` | Connection is on `keypair` with empty/invalid key | Recreate or PATCH connection with `"authType": "externalbrowser"` (local) or a valid PKCS#8 `privateKey` | + | `tool_prs` populated but `domain_prs` = 0 | Convertor subtask failed | Check pipeline subtask logs; ensure convertors are enabled | | Same repo also on GitHub API connection | Domain ID duplication risk | Remove the repo from one of the two connections | From 64fdf8176db0bf513c8f6af1320f6f2d172711da Mon Sep 17 00:00:00 2001 From: fmuntean Date: Mon, 3 Aug 2026 14:49:51 +0200 Subject: [PATCH 05/10] docs(github_snowflake): document Snowflake GITHUB_GROUP role checks Add console verification steps and troubleshooting for missing role grants before local plugin testing. Co-Authored-By: Cursor Grok 4.5 Co-authored-by: Cursor --- docs/github-snowflake-local-testing.md | 32 +++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/github-snowflake-local-testing.md b/docs/github-snowflake-local-testing.md index a9b9c41163e..9aa3a19c0dc 100644 --- a/docs/github-snowflake-local-testing.md +++ b/docs/github-snowflake-local-testing.md @@ -5,7 +5,9 @@ - Go 1.21+ - podman + podman-compose - Access to the Snowflake account (`GITHUB_DB.MARTS`) -- Snowflake role with `SELECT` on that schema (e.g. `GITHUB_GROUP`) +- Snowflake role `GITHUB_GROUP` **granted to your user** (SSO login alone is not enough — + ask a Snowflake admin if `USE ROLE GITHUB_GROUP` fails) +- Warehouse access (e.g. `DEFAULT`) for that role - A konflux-ci repo to pilot (`githubId` + `owner/repo` full name) --- @@ -14,6 +16,32 @@ Before starting DevLake, confirm you can reach Snowflake from your machine. +### 1a — Check role access in the Snowflake console + +In a Snowflake worksheet (use your real username from `SELECT CURRENT_USER()`, not the +literal string `CURRENT_USER`): + +```sql +SELECT CURRENT_USER(), CURRENT_ROLE(); + +-- Replace with the username returned above (quoted string, not CURRENT_USER as an identifier): +-- SHOW GRANTS TO USER "your.username"; + +USE ROLE GITHUB_GROUP; +USE WAREHOUSE DEFAULT; +USE DATABASE GITHUB_DB; +USE SCHEMA MARTS; + +SELECT COUNT(*) FROM REPOSITORY; +SELECT ID, FULL_NAME FROM REPOSITORY ORDER BY FULL_NAME LIMIT 20; +``` + +If `USE ROLE GITHUB_GROUP` fails with “not granted to this user”, stop and request the +role from a Snowflake admin before continuing. + + +### 1b — Check from Go (same auth path as the plugin) + Create a throwaway Go file (outside the repo): ```go @@ -78,8 +106,10 @@ SELECT ID, FULL_NAME FROM REPOSITORY ORDER BY FULL_NAME LIMIT 20; | Error | Fix | |---|---| | `account is empty` | Use the account identifier (e.g. `myorg-myaccount`), not the full `*.snowflakecomputing.com` URL | +| `Role 'GITHUB_GROUP' ... is not granted to this user` | Ask a Snowflake admin to grant `GITHUB_GROUP` to your user; verify with `USE ROLE GITHUB_GROUP` in a Snowflake worksheet | | `Object does not exist or not authorized` | Your role lacks `SELECT` on `GITHUB_DB.MARTS` — ask a Snowflake admin | | Browser window never opens | You may be running inside a container or headless SSH session — run locally | +| `User 'CURRENT_USER' does not exist` | `SHOW GRANTS TO USER` needs a real username string from `SELECT CURRENT_USER()`, not `CURRENT_USER` as an identifier | --- From 9967fe2d19ea974817fddb2b270a035b7537bbba Mon Sep 17 00:00:00 2001 From: fmuntean Date: Mon, 3 Aug 2026 15:40:32 +0200 Subject: [PATCH 06/10] fix(snowflake): harden private-key handling and share DB helpers Address Full Send medium findings by checking cursor.Err() in encrypt migrations and omitting PrivateKey from API JSON responses. Extract Open and ParseRSAPrivateKey into helpers/snowflakehelper for both github_snowflake and jira_snowflake. Co-Authored-By: Cursor Co-authored-by: Cursor --- backend/helpers/snowflakehelper/db.go | 95 +++++++++++++++++++ backend/helpers/snowflakehelper/db_test.go | 79 +++++++++++++++ backend/plugins/github_snowflake/AGENTS.md | 4 +- backend/plugins/github_snowflake/impl/impl.go | 3 +- .../github_snowflake/models/connection.go | 3 +- .../migrationscripts/encrypt_private_key.go | 3 + .../github_snowflake/tasks/task_data.go | 68 ------------- .../github_snowflake/tasks/task_data_test.go | 6 -- backend/plugins/jira_snowflake/AGENTS.md | 4 +- backend/plugins/jira_snowflake/impl/impl.go | 3 +- .../jira_snowflake/models/connection.go | 3 +- .../migrationscripts/encrypt_private_key.go | 3 + .../plugins/jira_snowflake/tasks/task_data.go | 68 ------------- .../jira_snowflake/tasks/task_data_test.go | 52 ---------- docs/upstream-diffs.md | 3 + 15 files changed, 197 insertions(+), 200 deletions(-) create mode 100644 backend/helpers/snowflakehelper/db.go create mode 100644 backend/helpers/snowflakehelper/db_test.go diff --git a/backend/helpers/snowflakehelper/db.go b/backend/helpers/snowflakehelper/db.go new file mode 100644 index 00000000000..7197417dd06 --- /dev/null +++ b/backend/helpers/snowflakehelper/db.go @@ -0,0 +1,95 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package snowflakehelper provides shared Snowflake connectivity helpers for +// Konflux-owned snowflake-backed DevLake plugins (github_snowflake, jira_snowflake). +package snowflakehelper + +import ( + "crypto/rsa" + "crypto/x509" + "database/sql" + "encoding/pem" + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + sf "github.com/snowflakedb/gosnowflake" +) + +// Open opens a database/sql connection to Snowflake. +// +// authType controls authentication: +// - "keypair" (default): JWT key-pair auth using privateKeyPEM. Works in containers and CI. +// - "externalbrowser": SSO via browser pop-up. Only works when DevLake runs on a desktop host +// (i.e. via `make run`, not inside a Docker container). +func Open(account, user, authType, privateKeyPEM, database, schema, warehouse, role string) (*sql.DB, errors.Error) { + cfg := &sf.Config{ + Account: account, + User: user, + Database: database, + Schema: schema, + } + if warehouse != "" { + cfg.Warehouse = warehouse + } + if role != "" { + cfg.Role = role + } + + switch authType { + case "", "keypair": + privKey, err := ParseRSAPrivateKey(privateKeyPEM) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") + } + cfg.Authenticator = sf.AuthTypeJwt + cfg.PrivateKey = privKey + case "externalbrowser": + cfg.Authenticator = sf.AuthTypeExternalBrowser + default: + return nil, errors.BadInput.New(fmt.Sprintf( + `unsupported authType %q; must be "keypair" or "externalbrowser"`, authType, + )) + } + + dsn, goErr := sf.DSN(cfg) + if goErr != nil { + return nil, errors.Default.Wrap(goErr, "failed to build Snowflake DSN") + } + db, goErr := sql.Open("snowflake", dsn) + if goErr != nil { + return nil, errors.Default.Wrap(goErr, "failed to open Snowflake connection") + } + return db, nil +} + +// ParseRSAPrivateKey parses a PKCS#8 PEM-encoded RSA private key. +func ParseRSAPrivateKey(pemStr string) (*rsa.PrivateKey, error) { + block, _ := pem.Decode([]byte(pemStr)) + if block == nil { + return nil, fmt.Errorf("failed to decode PEM block from private key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("failed to parse PKCS8 private key: %w", err) + } + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("private key is not an RSA key") + } + return rsaKey, nil +} diff --git a/backend/helpers/snowflakehelper/db_test.go b/backend/helpers/snowflakehelper/db_test.go new file mode 100644 index 00000000000..9065deffd2b --- /dev/null +++ b/backend/helpers/snowflakehelper/db_test.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package snowflakehelper + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func generateTestPKCS8PEM(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + der, err := x509.MarshalPKCS8PrivateKey(key) + require.NoError(t, err) + block := &pem.Block{Type: "PRIVATE KEY", Bytes: der} + return string(pem.EncodeToMemory(block)) +} + +func TestOpen_InvalidAuthType(t *testing.T) { + _, err := Open("acct", "user", "keypai", "", "db", "schema", "", "") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "unsupported authType") +} + +func TestParseRSAPrivateKey_ValidPKCS8(t *testing.T) { + pemStr := generateTestPKCS8PEM(t) + key, err := ParseRSAPrivateKey(pemStr) + require.NoError(t, err) + assert.NotNil(t, key) + assert.Equal(t, 2048, key.N.BitLen()) +} + +func TestParseRSAPrivateKey_EmptyString(t *testing.T) { + _, err := ParseRSAPrivateKey("") + assert.Error(t, err) + assert.Contains(t, err.Error(), "PEM") +} + +func TestParseRSAPrivateKey_InvalidPEM(t *testing.T) { + _, err := ParseRSAPrivateKey("not a pem block") + assert.Error(t, err) + assert.Contains(t, err.Error(), "PEM") +} + +func TestParseRSAPrivateKey_WrongKeyType(t *testing.T) { + // PKCS#1 format (BEGIN RSA PRIVATE KEY) is not PKCS#8, should fail ParsePKCS8PrivateKey + key, genErr := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, genErr) + block := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + } + pemStr := string(pem.EncodeToMemory(block)) + + _, err := ParseRSAPrivateKey(pemStr) + assert.Error(t, err, "PKCS#1 key should be rejected; expected PKCS#8") +} diff --git a/backend/plugins/github_snowflake/AGENTS.md b/backend/plugins/github_snowflake/AGENTS.md index a83bf36a4c4..ff06f584b9f 100644 --- a/backend/plugins/github_snowflake/AGENTS.md +++ b/backend/plugins/github_snowflake/AGENTS.md @@ -42,12 +42,14 @@ impl/impl.go — plugin interfaces, SubTaskMetas, PrepareTaskData api/connection_api.go — connection CRUD (POST/GET/PATCH/DELETE) models/connection.go — SnowflakeGithubConnection (table: _tool_github_snowflake_connections) models/migrationscripts/ — DB migrations -tasks/task_data.go — GithubSnowflakeOptions, GithubSnowflakeTaskData, OpenSnowflakeDB +tasks/task_data.go — GithubSnowflakeOptions, GithubSnowflakeTaskData tasks/shared.go — URL helpers tasks/sync_*.go — Snowflake SQL queries → _tool_github_* tool-layer tables tasks/convert_*.go — domain-layer convertors (adapted copies of github/tasks/*) ``` +Shared Snowflake connectivity lives in `helpers/snowflakehelper` (`Open`, `ParseRSAPrivateKey`). + ## Subtask pipeline order 1. `syncRepos` — REPOSITORY → `_tool_github_repos` diff --git a/backend/plugins/github_snowflake/impl/impl.go b/backend/plugins/github_snowflake/impl/impl.go index d283838d620..8d04ef38a8c 100644 --- a/backend/plugins/github_snowflake/impl/impl.go +++ b/backend/plugins/github_snowflake/impl/impl.go @@ -24,6 +24,7 @@ import ( coremodels "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/snowflakehelper" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" "github.com/apache/incubator-devlake/plugins/github_snowflake/api" "github.com/apache/incubator-devlake/plugins/github_snowflake/models" @@ -159,7 +160,7 @@ func (p GithubSnowflake) PrepareTaskData(taskCtx plugin.TaskContext, options map op.ScopeConfig = new(githubmodels.GithubScopeConfig) } - snowDB, openErr := tasks.OpenSnowflakeDB( + snowDB, openErr := snowflakehelper.Open( connection.Account, connection.User, connection.AuthType, diff --git a/backend/plugins/github_snowflake/models/connection.go b/backend/plugins/github_snowflake/models/connection.go index 5886c712b9a..f7f84c32326 100644 --- a/backend/plugins/github_snowflake/models/connection.go +++ b/backend/plugins/github_snowflake/models/connection.go @@ -35,7 +35,8 @@ type SnowflakeGithubConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - PrivateKey string `json:"privateKey" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` + // Omitted from JSON responses so GET/list never returns the decrypted PEM. + PrivateKey string `json:"-" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "GITHUB_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` // Schema is the Snowflake schema, e.g. "MARTS" diff --git a/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go index 86bf4dcf0a1..07ccefd0e43 100644 --- a/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go +++ b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go @@ -71,6 +71,9 @@ func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { return err } } + if err := cursor.Err(); err != nil { + return errors.Default.Wrap(err, "iterating connections cursor") + } return nil } diff --git a/backend/plugins/github_snowflake/tasks/task_data.go b/backend/plugins/github_snowflake/tasks/task_data.go index 95e9d491147..db3553a4f6c 100644 --- a/backend/plugins/github_snowflake/tasks/task_data.go +++ b/backend/plugins/github_snowflake/tasks/task_data.go @@ -18,17 +18,13 @@ limitations under the License. package tasks import ( - "crypto/rsa" - "crypto/x509" "database/sql" - "encoding/pem" "fmt" "strings" "github.com/apache/incubator-devlake/core/errors" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" - sf "github.com/snowflakedb/gosnowflake" ) // Raw table name constants used for StatefulDataConverter state params @@ -102,70 +98,6 @@ func validateOwnerRepo(name string) errors.Error { return nil } -// OpenSnowflakeDB opens a database/sql connection to Snowflake. -// -// authType controls authentication: -// - "keypair" (default): JWT key-pair auth using privateKeyPEM. Works in containers and CI. -// - "externalbrowser": SSO via browser pop-up. Only works when DevLake runs on a desktop host -// (i.e. via `make run`, not inside a Docker container). -func OpenSnowflakeDB(account, user, authType, privateKeyPEM, database, schema, warehouse, role string) (*sql.DB, errors.Error) { - cfg := &sf.Config{ - Account: account, - User: user, - Database: database, - Schema: schema, - } - if warehouse != "" { - cfg.Warehouse = warehouse - } - if role != "" { - cfg.Role = role - } - - switch authType { - case "", "keypair": - privKey, err := parseRSAPrivateKey(privateKeyPEM) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") - } - cfg.Authenticator = sf.AuthTypeJwt - cfg.PrivateKey = privKey - case "externalbrowser": - cfg.Authenticator = sf.AuthTypeExternalBrowser - default: - return nil, errors.BadInput.New(fmt.Sprintf( - `unsupported authType %q; must be "keypair" or "externalbrowser"`, authType, - )) - } - - dsn, goErr := sf.DSN(cfg) - if goErr != nil { - return nil, errors.Default.Wrap(goErr, "failed to build Snowflake DSN") - } - db, goErr := sql.Open("snowflake", dsn) - if goErr != nil { - return nil, errors.Default.Wrap(goErr, "failed to open Snowflake connection") - } - return db, nil -} - -// parseRSAPrivateKey parses a PKCS#8 PEM-encoded RSA private key. -func parseRSAPrivateKey(pemStr string) (*rsa.PrivateKey, error) { - block, _ := pem.Decode([]byte(pemStr)) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block from private key") - } - key, err := x509.ParsePKCS8PrivateKey(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse PKCS8 private key: %w", err) - } - rsaKey, ok := key.(*rsa.PrivateKey) - if !ok { - return nil, fmt.Errorf("private key is not an RSA key") - } - return rsaKey, nil -} - // RepoShortName returns the short name from owner/repo full name. func RepoShortName(fullName string) string { parts := strings.Split(fullName, "/") diff --git a/backend/plugins/github_snowflake/tasks/task_data_test.go b/backend/plugins/github_snowflake/tasks/task_data_test.go index 572e5110794..4d771a1993b 100644 --- a/backend/plugins/github_snowflake/tasks/task_data_test.go +++ b/backend/plugins/github_snowflake/tasks/task_data_test.go @@ -92,9 +92,3 @@ func TestDecodeAndValidateTaskOptions_InvalidNameFormat(t *testing.T) { }) assert.NotNil(t, err) } - -func TestOpenSnowflakeDB_InvalidAuthType(t *testing.T) { - _, err := OpenSnowflakeDB("acct", "user", "keypai", "", "db", "schema", "", "") - assert.NotNil(t, err) - assert.Contains(t, err.Error(), "unsupported authType") -} diff --git a/backend/plugins/jira_snowflake/AGENTS.md b/backend/plugins/jira_snowflake/AGENTS.md index ff32c088f27..44fbe14f292 100644 --- a/backend/plugins/jira_snowflake/AGENTS.md +++ b/backend/plugins/jira_snowflake/AGENTS.md @@ -21,12 +21,14 @@ impl/impl.go — plugin interfaces, SubTaskMetas, PrepareTaskData api/connection_api.go — connection CRUD (POST/GET/PATCH/DELETE) models/connection.go — SnowflakeJiraConnection (table: _tool_jira_snowflake_connections) models/migrationscripts/ — DB migrations -tasks/task_data.go — JiraSnowflakeOptions, JiraSnowflakeTaskData, OpenSnowflakeDB +tasks/task_data.go — JiraSnowflakeOptions, JiraSnowflakeTaskData tasks/shared.go — getStdStatus, getTypeMappings (adapted from jira/tasks) tasks/sync_*.go — Snowflake SQL queries → _tool_jira_* tool-layer tables tasks/convert_*.go — domain-layer convertors (adapted copies of jira/tasks/*) ``` +Shared Snowflake connectivity lives in `helpers/snowflakehelper` (`Open`, `ParseRSAPrivateKey`). + ## Subtask pipeline order 1. `syncIssues` — Snowflake JIRA_ISSUE_NON_PII → `_tool_jira_issues` + `_tool_jira_board_issues` diff --git a/backend/plugins/jira_snowflake/impl/impl.go b/backend/plugins/jira_snowflake/impl/impl.go index 40e560e6f5b..7e9dcd42c46 100644 --- a/backend/plugins/jira_snowflake/impl/impl.go +++ b/backend/plugins/jira_snowflake/impl/impl.go @@ -24,6 +24,7 @@ import ( coremodels "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/snowflakehelper" jiramodels "github.com/apache/incubator-devlake/plugins/jira/models" "github.com/apache/incubator-devlake/plugins/jira_snowflake/api" "github.com/apache/incubator-devlake/plugins/jira_snowflake/models" @@ -160,7 +161,7 @@ func (p JiraSnowflake) PrepareTaskData(taskCtx plugin.TaskContext, options map[s } // Open the Snowflake SQL connection (closed in Close()) - snowDB, openErr := tasks.OpenSnowflakeDB( + snowDB, openErr := snowflakehelper.Open( connection.Account, connection.User, connection.AuthType, diff --git a/backend/plugins/jira_snowflake/models/connection.go b/backend/plugins/jira_snowflake/models/connection.go index 5d96993ae79..8e83ff89dc4 100644 --- a/backend/plugins/jira_snowflake/models/connection.go +++ b/backend/plugins/jira_snowflake/models/connection.go @@ -35,7 +35,8 @@ type SnowflakeJiraConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - PrivateKey string `json:"privateKey" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` + // Omitted from JSON responses so GET/list never returns the decrypted PEM. + PrivateKey string `json:"-" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "JIRA_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` // Schema is the Snowflake schema, e.g. "CLOUDRHAI_MARTS" diff --git a/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go index 7e1e9d39e66..8f93bd60765 100644 --- a/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go +++ b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go @@ -71,6 +71,9 @@ func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { return err } } + if err := cursor.Err(); err != nil { + return errors.Default.Wrap(err, "iterating connections cursor") + } return nil } diff --git a/backend/plugins/jira_snowflake/tasks/task_data.go b/backend/plugins/jira_snowflake/tasks/task_data.go index 831f40ac81f..1943a46039c 100644 --- a/backend/plugins/jira_snowflake/tasks/task_data.go +++ b/backend/plugins/jira_snowflake/tasks/task_data.go @@ -18,16 +18,12 @@ limitations under the License. package tasks import ( - "crypto/rsa" - "crypto/x509" "database/sql" - "encoding/pem" "fmt" "github.com/apache/incubator-devlake/core/errors" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" jiramodels "github.com/apache/incubator-devlake/plugins/jira/models" - sf "github.com/snowflakedb/gosnowflake" ) // JiraSnowflakeOptions contains all per-pipeline task options. @@ -71,67 +67,3 @@ func DecodeAndValidateTaskOptions(options map[string]interface{}) (*JiraSnowflak } return &op, nil } - -// OpenSnowflakeDB opens a database/sql connection to Snowflake. -// -// authType controls authentication: -// - "keypair" (default): JWT key-pair auth using privateKeyPEM. Works in containers and CI. -// - "externalbrowser": SSO via browser pop-up. Only works when DevLake runs on a desktop host -// (i.e. via `make run`, not inside a Docker container). -func OpenSnowflakeDB(account, user, authType, privateKeyPEM, database, schema, warehouse, role string) (*sql.DB, errors.Error) { - cfg := &sf.Config{ - Account: account, - User: user, - Database: database, - Schema: schema, - } - if warehouse != "" { - cfg.Warehouse = warehouse - } - if role != "" { - cfg.Role = role - } - - switch authType { - case "", "keypair": - privKey, err := parseRSAPrivateKey(privateKeyPEM) - if err != nil { - return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") - } - cfg.Authenticator = sf.AuthTypeJwt - cfg.PrivateKey = privKey - case "externalbrowser": - cfg.Authenticator = sf.AuthTypeExternalBrowser - default: - return nil, errors.BadInput.New(fmt.Sprintf( - `unsupported authType %q; must be "keypair" or "externalbrowser"`, authType, - )) - } - - dsn, goErr := sf.DSN(cfg) - if goErr != nil { - return nil, errors.Default.Wrap(goErr, "failed to build Snowflake DSN") - } - db, goErr := sql.Open("snowflake", dsn) - if goErr != nil { - return nil, errors.Default.Wrap(goErr, "failed to open Snowflake connection") - } - return db, nil -} - -// parseRSAPrivateKey parses a PKCS#8 PEM-encoded RSA private key. -func parseRSAPrivateKey(pemStr string) (*rsa.PrivateKey, error) { - block, _ := pem.Decode([]byte(pemStr)) - if block == nil { - return nil, fmt.Errorf("failed to decode PEM block from private key") - } - key, err := x509.ParsePKCS8PrivateKey(block.Bytes) - if err != nil { - return nil, fmt.Errorf("failed to parse PKCS8 private key: %w", err) - } - rsaKey, ok := key.(*rsa.PrivateKey) - if !ok { - return nil, fmt.Errorf("private key is not an RSA key") - } - return rsaKey, nil -} diff --git a/backend/plugins/jira_snowflake/tasks/task_data_test.go b/backend/plugins/jira_snowflake/tasks/task_data_test.go index 42e7c8ef16c..3b5d752c613 100644 --- a/backend/plugins/jira_snowflake/tasks/task_data_test.go +++ b/backend/plugins/jira_snowflake/tasks/task_data_test.go @@ -18,10 +18,6 @@ limitations under the License. package tasks import ( - "crypto/rand" - "crypto/rsa" - "crypto/x509" - "encoding/pem" "testing" "github.com/stretchr/testify/assert" @@ -81,54 +77,6 @@ func TestDecodeAndValidateTaskOptions_MissingProjectKeys(t *testing.T) { assert.Contains(t, err.Error(), "projectKeys") } -// --------------------------------------------------------------------------- -// parseRSAPrivateKey -// --------------------------------------------------------------------------- - -func generateTestPKCS8PEM(t *testing.T) string { - t.Helper() - key, err := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, err) - der, err := x509.MarshalPKCS8PrivateKey(key) - require.NoError(t, err) - block := &pem.Block{Type: "PRIVATE KEY", Bytes: der} - return string(pem.EncodeToMemory(block)) -} - -func TestParseRSAPrivateKey_ValidPKCS8(t *testing.T) { - pemStr := generateTestPKCS8PEM(t) - key, err := parseRSAPrivateKey(pemStr) - require.NoError(t, err) - assert.NotNil(t, key) - assert.Equal(t, 2048, key.N.BitLen()) -} - -func TestParseRSAPrivateKey_EmptyString(t *testing.T) { - _, err := parseRSAPrivateKey("") - assert.Error(t, err) - assert.Contains(t, err.Error(), "PEM") -} - -func TestParseRSAPrivateKey_InvalidPEM(t *testing.T) { - _, err := parseRSAPrivateKey("not a pem block") - assert.Error(t, err) - assert.Contains(t, err.Error(), "PEM") -} - -func TestParseRSAPrivateKey_WrongKeyType(t *testing.T) { - // PKCS#1 format (BEGIN RSA PRIVATE KEY) is not PKCS#8, should fail ParsePKCS8PrivateKey - key, genErr := rsa.GenerateKey(rand.Reader, 2048) - require.NoError(t, genErr) - block := &pem.Block{ - Type: "RSA PRIVATE KEY", - Bytes: x509.MarshalPKCS1PrivateKey(key), - } - pemStr := string(pem.EncodeToMemory(block)) - - _, err := parseRSAPrivateKey(pemStr) - assert.Error(t, err, "PKCS#1 key should be rejected; expected PKCS#8") -} - // --------------------------------------------------------------------------- // stringVal // --------------------------------------------------------------------------- diff --git a/docs/upstream-diffs.md b/docs/upstream-diffs.md index f7ac4d772d1..ae20642b038 100644 --- a/docs/upstream-diffs.md +++ b/docs/upstream-diffs.md @@ -6,6 +6,9 @@ that must be maintained during upstream syncs. Owned plugins (`aireview`, `codecov`, `testregistry`, `agentready`, `langfuse`, `jira_snowflake`, `github_snowflake`) are additions, not modifications, and are not tracked here. +Shared helper packages added for owned plugins (e.g. `backend/helpers/snowflakehelper/`) are also +fork additions and are not tracked as file-level upstream diffs below. + `jira_snowflake/tasks/convert_*.go` are adapted copies of `jira/tasks/` convertors — see the [jira_snowflake AGENTS.md](../backend/plugins/jira_snowflake/AGENTS.md) for the diff details. From 0a255dd4d9eb6ee125f6c7eef3151c8346158a6e Mon Sep 17 00:00:00 2001 From: fmuntean Date: Tue, 4 Aug 2026 16:13:57 +0200 Subject: [PATCH 07/10] fix(github_snowflake): fix reviewer removal and sync edge cases Filter REMOVED after QUALIFY so later removals win over older requests. Use COALESCE for PR incremental time filters, and skip already-encrypted private keys in snowflake encrypt migrations. Co-Authored-By: Cursor Co-authored-by: Cursor --- .../github_snowflake/models/connection.go | 3 +- .../migrationscripts/encrypt_private_key.go | 5 ++ .../tasks/sync_pull_requests.go | 4 +- .../tasks/sync_pull_requests_test.go | 2 +- .../github_snowflake/tasks/sync_reviewers.go | 50 +++++++++++-------- .../tasks/sync_reviewers_test.go | 4 +- .../jira_snowflake/models/connection.go | 3 +- .../migrationscripts/encrypt_private_key.go | 5 ++ 8 files changed, 51 insertions(+), 25 deletions(-) diff --git a/backend/plugins/github_snowflake/models/connection.go b/backend/plugins/github_snowflake/models/connection.go index f7f84c32326..dc598b59309 100644 --- a/backend/plugins/github_snowflake/models/connection.go +++ b/backend/plugins/github_snowflake/models/connection.go @@ -35,7 +35,8 @@ type SnowflakeGithubConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - // Omitted from JSON responses so GET/list never returns the decrypted PEM. + // json:"-" omits the decrypted PEM from GET/list responses. Request binding + // still works via mapstructure:"privateKey" (ConnectionApiHelper.Decode). PrivateKey string `json:"-" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "GITHUB_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` diff --git a/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go index 07ccefd0e43..f71b7eea664 100644 --- a/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go +++ b/backend/plugins/github_snowflake/models/migrationscripts/encrypt_private_key.go @@ -58,6 +58,11 @@ func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { if row.PrivateKey == "" { continue } + // Skip values that are already encrypted so a migration replay cannot + // double-encrypt and permanently corrupt the private key. + if _, decErr := plugin.Decrypt(encKey, row.PrivateKey); decErr == nil { + continue + } encrypted, err := plugin.Encrypt(encKey, row.PrivateKey) if err != nil { return err diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go index 09882601ea4..aed8e6f33cd 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go @@ -177,7 +177,9 @@ WHERE i.REPOSITORY_ID = ? ` args := []interface{}{repoId} if timeAfter != nil { - query += " AND pr.UPDATED_AT > ?\n" + // UPDATED_AT can be NULL for older PRs; fall back to created timestamps + // so incremental sync does not silently drop them. + query += " AND COALESCE(pr.UPDATED_AT, COALESCE(pr.CREATED_AT, i.CREATED_AT)) > ?\n" args = append(args, *timeAfter) } return query, args diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go index 4f6fc754b2c..06334ecf6f4 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests_test.go @@ -38,7 +38,7 @@ func TestBuildPullRequestsQuery_NoTimeFilter(t *testing.T) { func TestBuildPullRequestsQuery_WithTimeFilter(t *testing.T) { ts := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC) q, args := buildPullRequestsQuery(12345, &ts) - assert.Contains(t, q, "AND pr.UPDATED_AT > ?") + assert.Contains(t, q, "AND COALESCE(pr.UPDATED_AT, COALESCE(pr.CREATED_AT, i.CREATED_AT)) > ?") assert.Equal(t, 2, len(args)) assert.Equal(t, 12345, args[0]) assert.Equal(t, ts, args[1]) diff --git a/backend/plugins/github_snowflake/tasks/sync_reviewers.go b/backend/plugins/github_snowflake/tasks/sync_reviewers.go index b1d480417d1..cd5089ce84d 100644 --- a/backend/plugins/github_snowflake/tasks/sync_reviewers.go +++ b/backend/plugins/github_snowflake/tasks/sync_reviewers.go @@ -89,34 +89,44 @@ func SyncReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { } func buildReviewersQuery(repoId int, timeAfter *time.Time) (string, []interface{}) { - // Latest non-removed user request per (pull_request_id, requested_id). + // First pick the latest history row per (pull_request_id, requested_id), then + // drop rows whose latest event is a removal. Filtering REMOVED before QUALIFY + // incorrectly keeps an older "requested" row when a later removal exists. query := ` SELECT - h.REQUESTED_ID AS reviewer_id, - h.PULL_REQUEST_ID AS pull_request_id, - u.LOGIN AS username, - u.NAME AS name -FROM REQUESTED_REVIEWER_HISTORY h -JOIN PULL_REQUEST pr - ON pr.ID = h.PULL_REQUEST_ID -JOIN ISSUE i - ON i.ID = pr.ISSUE_ID -LEFT JOIN "USER" u - ON u.ID = h.REQUESTED_ID -WHERE i.REPOSITORY_ID = ? - AND LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user' - AND (h.REMOVED IS NULL OR h.REMOVED = FALSE) + latest.reviewer_id, + latest.pull_request_id, + latest.username, + latest.name +FROM ( + SELECT + h.REQUESTED_ID AS reviewer_id, + h.PULL_REQUEST_ID AS pull_request_id, + u.LOGIN AS username, + u.NAME AS name, + h.REMOVED AS removed + FROM REQUESTED_REVIEWER_HISTORY h + JOIN PULL_REQUEST pr + ON pr.ID = h.PULL_REQUEST_ID + JOIN ISSUE i + ON i.ID = pr.ISSUE_ID + LEFT JOIN "USER" u + ON u.ID = h.REQUESTED_ID + WHERE i.REPOSITORY_ID = ? + AND LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user' ` args := []interface{}{repoId} if timeAfter != nil { - query += " AND h.CREATED_AT > ?\n" + query += " AND h.CREATED_AT > ?\n" args = append(args, *timeAfter) } query += ` -QUALIFY ROW_NUMBER() OVER ( - PARTITION BY h.PULL_REQUEST_ID, h.REQUESTED_ID - ORDER BY h.CREATED_AT DESC NULLS LAST -) = 1 + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY h.PULL_REQUEST_ID, h.REQUESTED_ID + ORDER BY h.CREATED_AT DESC NULLS LAST + ) = 1 +) latest +WHERE (latest.removed IS NULL OR latest.removed = FALSE) ` return query, args } diff --git a/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go b/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go index d677fd995ff..768673264df 100644 --- a/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go +++ b/backend/plugins/github_snowflake/tasks/sync_reviewers_test.go @@ -30,7 +30,9 @@ func TestBuildReviewersQuery_FiltersUsersAndLatest(t *testing.T) { assert.Contains(t, q, "LOWER(h.REQUESTED_REVIEWER_TYPE) = 'user'") assert.Contains(t, q, "QUALIFY ROW_NUMBER()") assert.Contains(t, q, "PARTITION BY h.PULL_REQUEST_ID, h.REQUESTED_ID") - assert.Contains(t, q, "h.REMOVED IS NULL OR h.REMOVED = FALSE") + // REMOVED must be filtered after QUALIFY so a later removal wins. + assert.Contains(t, q, "WHERE (latest.removed IS NULL OR latest.removed = FALSE)") + assert.NotContains(t, q, "AND (h.REMOVED IS NULL OR h.REMOVED = FALSE)") assert.Equal(t, []interface{}{42}, args) } diff --git a/backend/plugins/jira_snowflake/models/connection.go b/backend/plugins/jira_snowflake/models/connection.go index 8e83ff89dc4..4ec1354118f 100644 --- a/backend/plugins/jira_snowflake/models/connection.go +++ b/backend/plugins/jira_snowflake/models/connection.go @@ -35,7 +35,8 @@ type SnowflakeJiraConnection struct { AuthType string `json:"authType" gorm:"column:auth_type;default:keypair" mapstructure:"authType"` // PrivateKey is the RSA private key in PKCS#8 PEM format, stored encrypted. // Required when AuthType is "keypair"; ignored for "externalbrowser". - // Omitted from JSON responses so GET/list never returns the decrypted PEM. + // json:"-" omits the decrypted PEM from GET/list responses. Request binding + // still works via mapstructure:"privateKey" (ConnectionApiHelper.Decode). PrivateKey string `json:"-" gorm:"column:private_key;type:text;serializer:encdec" mapstructure:"privateKey"` // Database is the Snowflake database, e.g. "JIRA_DB" Database string `json:"database" gorm:"column:sf_database;not null" mapstructure:"database" validate:"required"` diff --git a/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go index 8f93bd60765..20ed247268f 100644 --- a/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go +++ b/backend/plugins/jira_snowflake/models/migrationscripts/encrypt_private_key.go @@ -58,6 +58,11 @@ func (*encryptPrivateKey) Up(basicRes context.BasicRes) errors.Error { if row.PrivateKey == "" { continue } + // Skip values that are already encrypted so a migration replay cannot + // double-encrypt and permanently corrupt the private key. + if _, decErr := plugin.Decrypt(encKey, row.PrivateKey); decErr == nil { + continue + } encrypted, err := plugin.Encrypt(encKey, row.PrivateKey) if err != nil { return err From cf678604af4e7354939265c4bd9990f0243c5c39 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Thu, 6 Aug 2026 11:50:44 +0200 Subject: [PATCH 08/10] fix(github_snowflake): address Full Send medium review findings Reject empty private keys for keypair auth with a clear error, and populate RawDataOrigin on sync writes so full-sync domain deletes by _raw_data_params match. Co-Authored-By: Cursor Co-authored-by: Cursor --- backend/helpers/snowflakehelper/db.go | 3 +++ backend/helpers/snowflakehelper/db_test.go | 11 +++++++++++ backend/plugins/github_snowflake/AGENTS.md | 6 ++++-- backend/plugins/github_snowflake/tasks/shared.go | 15 +++++++++++++++ .../plugins/github_snowflake/tasks/shared_test.go | 8 ++++++++ .../github_snowflake/tasks/sync_pr_commits.go | 4 ++-- .../github_snowflake/tasks/sync_pr_reviews.go | 4 ++-- .../github_snowflake/tasks/sync_pull_requests.go | 3 +-- .../github_snowflake/tasks/sync_reviewers.go | 4 ++-- 9 files changed, 48 insertions(+), 10 deletions(-) diff --git a/backend/helpers/snowflakehelper/db.go b/backend/helpers/snowflakehelper/db.go index 7197417dd06..265aae713ab 100644 --- a/backend/helpers/snowflakehelper/db.go +++ b/backend/helpers/snowflakehelper/db.go @@ -52,6 +52,9 @@ func Open(account, user, authType, privateKeyPEM, database, schema, warehouse, r switch authType { case "", "keypair": + if privateKeyPEM == "" { + return nil, errors.BadInput.New("privateKey is required for keypair auth") + } privKey, err := ParseRSAPrivateKey(privateKeyPEM) if err != nil { return nil, errors.Default.Wrap(err, "failed to parse Snowflake private key") diff --git a/backend/helpers/snowflakehelper/db_test.go b/backend/helpers/snowflakehelper/db_test.go index 9065deffd2b..dab90bf79c2 100644 --- a/backend/helpers/snowflakehelper/db_test.go +++ b/backend/helpers/snowflakehelper/db_test.go @@ -22,6 +22,7 @@ import ( "crypto/rsa" "crypto/x509" "encoding/pem" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -44,6 +45,16 @@ func TestOpen_InvalidAuthType(t *testing.T) { assert.Contains(t, err.Error(), "unsupported authType") } +func TestOpen_EmptyPrivateKeyForKeypair(t *testing.T) { + for _, authType := range []string{"", "keypair"} { + t.Run(fmt.Sprintf("authType=%q", authType), func(t *testing.T) { + _, err := Open("acct", "user", authType, "", "db", "schema", "", "") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "privateKey is required for keypair auth") + }) + } +} + func TestParseRSAPrivateKey_ValidPKCS8(t *testing.T) { pemStr := generateTestPKCS8PEM(t) key, err := ParseRSAPrivateKey(pemStr) diff --git a/backend/plugins/github_snowflake/AGENTS.md b/backend/plugins/github_snowflake/AGENTS.md index ff06f584b9f..adc69ed94a7 100644 --- a/backend/plugins/github_snowflake/AGENTS.md +++ b/backend/plugins/github_snowflake/AGENTS.md @@ -63,8 +63,10 @@ Shared Snowflake connectivity lives in `helpers/snowflakehelper` (`Open`, `Parse ## Key conventions - **Scope unit is `GithubRepo`** (numeric `githubId` + `fullName` owner/repo). -- **No raw-table layer**: writes directly to `_tool_github_*`. Convertors use - `_raw_data_params`-scoped deletion for full sync (same pattern as jira_snowflake). +- **No raw-table layer**: writes directly to `_tool_github_*`. Sync tasks populate + `RawDataOrigin` (`_raw_data_table` / `_raw_data_params`) on tool-layer rows so + convertors can delete domain records by `_raw_data_params` on full sync + (same pattern as jira_snowflake). - **AuthType**: `"keypair"` (default, JWT) or `"externalbrowser"` (SSO, desktop only). - **Connection defaults**: Database=`GITHUB_DB`, Schema=`MARTS`, Warehouse=`DEFAULT`. - **PR line/comment counts** are unavailable in Snowflake — leave 0. diff --git a/backend/plugins/github_snowflake/tasks/shared.go b/backend/plugins/github_snowflake/tasks/shared.go index 301aa4bbe67..fde6934753a 100644 --- a/backend/plugins/github_snowflake/tasks/shared.go +++ b/backend/plugins/github_snowflake/tasks/shared.go @@ -19,6 +19,9 @@ package tasks import ( "fmt" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/utils" ) // deriveRepoHTMLUrl builds the GitHub HTML URL from FULL_NAME. @@ -36,6 +39,18 @@ func derivePullRequestURL(fullName string, number int) string { return fmt.Sprintf("https://github.com/%s/pull/%d", fullName, number) } +// toolLayerNoPKModel builds a NoPKModel with RawDataOrigin fields that match +// StatefulDataConverter params so full-sync domain deletes by _raw_data_params work. +func toolLayerNoPKModel(rawTable string, connectionId uint64, name string) common.NoPKModel { + m := common.NewNoPKModel() + m.RawDataTable = "_raw_" + rawTable + m.RawDataParams = utils.ToJsonString(GithubApiParams{ + ConnectionId: connectionId, + Name: name, + }) + return m +} + // nullStr returns the string value or empty string when nil. func nullStr(s *string) string { if s == nil { diff --git a/backend/plugins/github_snowflake/tasks/shared_test.go b/backend/plugins/github_snowflake/tasks/shared_test.go index 9b84bb7da9c..bfc2fd43def 100644 --- a/backend/plugins/github_snowflake/tasks/shared_test.go +++ b/backend/plugins/github_snowflake/tasks/shared_test.go @@ -51,3 +51,11 @@ func TestRepoShortName(t *testing.T) { assert.Equal(t, "build-service", RepoShortName("konflux-ci/build-service")) assert.Equal(t, "alone", RepoShortName("alone")) } + +func TestToolLayerNoPKModel(t *testing.T) { + m := toolLayerNoPKModel(RAW_PULL_REQUEST_TABLE, 7, "konflux-ci/build-service") + assert.Equal(t, "_raw_github_api_pull_requests", m.RawDataTable) + assert.Equal(t, `{"ConnectionId":7,"Name":"konflux-ci/build-service"}`, m.RawDataParams) + assert.False(t, m.CreatedAt.IsZero()) + assert.False(t, m.UpdatedAt.IsZero()) +} diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_commits.go b/backend/plugins/github_snowflake/tasks/sync_pr_commits.go index e8df6412a65..735c32eb6a4 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pr_commits.go +++ b/backend/plugins/github_snowflake/tasks/sync_pr_commits.go @@ -21,7 +21,6 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" ) @@ -41,6 +40,7 @@ func SyncPrCommits(subtaskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId repoId := data.Options.GithubId + fullName := data.Options.Name var timeAfter *time.Time syncPolicy := subtaskCtx.TaskContext().SyncPolicy() @@ -75,7 +75,7 @@ func SyncPrCommits(subtaskCtx plugin.SubTaskContext) errors.Error { CommitAuthorName: nullStr(authorName), CommitAuthorEmail: nullStr(authorEmail), CommitAuthoredDate: authoredDate, - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_PR_COMMIT_TABLE, connectionId, fullName), } if dbErr := db.CreateOrUpdate(prCommit); dbErr != nil { return dbErr diff --git a/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go b/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go index 8438908c612..50187bc1794 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go +++ b/backend/plugins/github_snowflake/tasks/sync_pr_reviews.go @@ -21,7 +21,6 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" ) @@ -41,6 +40,7 @@ func SyncPrReviews(subtaskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId repoId := data.Options.GithubId + fullName := data.Options.Name var timeAfter *time.Time syncPolicy := subtaskCtx.TaskContext().SyncPolicy() @@ -84,7 +84,7 @@ func SyncPrReviews(subtaskCtx plugin.SubTaskContext) errors.Error { GithubSubmitAt: submittedAt, AuthorUserId: nullInt(authorUserId), AuthorUsername: nullStr(authorLogin), - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_PR_REVIEW_TABLE, connectionId, fullName), } if dbErr := db.CreateOrUpdate(review); dbErr != nil { return dbErr diff --git a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go index aed8e6f33cd..11b715510f4 100644 --- a/backend/plugins/github_snowflake/tasks/sync_pull_requests.go +++ b/backend/plugins/github_snowflake/tasks/sync_pull_requests.go @@ -21,7 +21,6 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" ) @@ -125,7 +124,7 @@ func SyncPullRequests(subtaskCtx plugin.SubTaskContext) errors.Error { MergedById: nullInt(mergedById), MergedByName: nullStr(mergedByName), Url: derivePullRequestURL(fullName, int(number)), - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_PULL_REQUEST_TABLE, connectionId, fullName), } if dbErr := db.CreateOrUpdate(pr); dbErr != nil { return dbErr diff --git a/backend/plugins/github_snowflake/tasks/sync_reviewers.go b/backend/plugins/github_snowflake/tasks/sync_reviewers.go index cd5089ce84d..cf388deca22 100644 --- a/backend/plugins/github_snowflake/tasks/sync_reviewers.go +++ b/backend/plugins/github_snowflake/tasks/sync_reviewers.go @@ -21,7 +21,6 @@ import ( "time" "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" ) @@ -41,6 +40,7 @@ func SyncReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId repoId := data.Options.GithubId + fullName := data.Options.Name var timeAfter *time.Time syncPolicy := subtaskCtx.TaskContext().SyncPolicy() @@ -73,7 +73,7 @@ func SyncReviewers(subtaskCtx plugin.SubTaskContext) errors.Error { PullRequestId: int(pullRequestId), Username: nullStr(username), Name: nullStr(name), - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_PR_REVIEW_TABLE, connectionId, fullName), } if dbErr := db.CreateOrUpdate(reviewer); dbErr != nil { return dbErr From 0f1041e1c328b9afd21e69e889aa635868cadf0b Mon Sep 17 00:00:00 2001 From: fmuntean Date: Thu, 6 Aug 2026 13:28:16 +0200 Subject: [PATCH 09/10] fix(github_snowflake): populate RawDataOrigin on account sync Align SyncAccounts with other sync tasks so tool-layer account rows carry _raw_data_params for full-sync domain cleanup. Co-Authored-By: Cursor Co-authored-by: Cursor --- backend/plugins/github_snowflake/tasks/sync_accounts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/plugins/github_snowflake/tasks/sync_accounts.go b/backend/plugins/github_snowflake/tasks/sync_accounts.go index 07afdea070a..364d10a4277 100644 --- a/backend/plugins/github_snowflake/tasks/sync_accounts.go +++ b/backend/plugins/github_snowflake/tasks/sync_accounts.go @@ -19,7 +19,6 @@ package tasks import ( "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/plugin" githubmodels "github.com/apache/incubator-devlake/plugins/github/models" ) @@ -39,6 +38,7 @@ func SyncAccounts(subtaskCtx plugin.SubTaskContext) errors.Error { connectionId := data.Options.ConnectionId repoId := data.Options.GithubId + fullName := data.Options.Name query, args := buildAccountsQuery(repoId) rows, goErr := data.SnowflakeDB.QueryContext(subtaskCtx.GetContext(), query, args...) @@ -66,7 +66,7 @@ func SyncAccounts(subtaskCtx plugin.SubTaskContext) errors.Error { Login: loginStr, Name: nullStr(name), Email: nullStr(email), - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_ACCOUNT_TABLE, connectionId, fullName), } if loginStr != "" { account.HtmlUrl = "https://github.com/" + loginStr @@ -81,7 +81,7 @@ func SyncAccounts(subtaskCtx plugin.SubTaskContext) errors.Error { AccountId: int(id), RepoGithubId: repoId, Login: loginStr, - NoPKModel: common.NewNoPKModel(), + NoPKModel: toolLayerNoPKModel(RAW_ACCOUNT_TABLE, connectionId, fullName), } if dbErr := db.CreateOrUpdate(repoAccount); dbErr != nil { return dbErr From c05c649d7bb64664f7cf72b60c54f89e233de5b7 Mon Sep 17 00:00:00 2001 From: fmuntean Date: Wed, 19 Aug 2026 12:26:48 +0200 Subject: [PATCH 10/10] fix(github_snowflake): qualify account convertor join and resolve upstream-diffs conflict Make ConvertAccounts filter on gra.repo_github_id with an INNER JOIN so the query is unambiguous. Keep both the snowflakehelper and pkg/ notes when resolving the merge conflict in docs/upstream-diffs.md. Co-Authored-By: Cursor Grok 4.6 Co-authored-by: Cursor --- backend/plugins/github_snowflake/tasks/convert_accounts.go | 4 ++-- docs/upstream-diffs.md | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backend/plugins/github_snowflake/tasks/convert_accounts.go b/backend/plugins/github_snowflake/tasks/convert_accounts.go index d0fa529451f..2eaec2d600e 100644 --- a/backend/plugins/github_snowflake/tasks/convert_accounts.go +++ b/backend/plugins/github_snowflake/tasks/convert_accounts.go @@ -61,11 +61,11 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { dal.Select("_tool_github_accounts.*"), dal.From(&githubmodels.GithubAccount{}), dal.Where( - "repo_github_id = ? and _tool_github_accounts.connection_id=?", + "gra.repo_github_id = ? AND _tool_github_accounts.connection_id=?", data.Options.GithubId, data.Options.ConnectionId, ), - dal.Join(`left join _tool_github_repo_accounts gra on ( + dal.Join(`inner join _tool_github_repo_accounts gra on ( _tool_github_accounts.connection_id = gra.connection_id AND _tool_github_accounts.id = gra.account_id )`), diff --git a/docs/upstream-diffs.md b/docs/upstream-diffs.md index b8ef06b5b37..e7a445906cc 100644 --- a/docs/upstream-diffs.md +++ b/docs/upstream-diffs.md @@ -6,14 +6,12 @@ that must be maintained during upstream syncs. Owned plugins (`aireview`, `codecov`, `testregistry`, `agentready`, `langfuse`, `jira_snowflake`, `github_snowflake`) are additions, not modifications, and are not tracked here. -<<<<<<< HEAD Shared helper packages added for owned plugins (e.g. `backend/helpers/snowflakehelper/`) are also fork additions and are not tracked as file-level upstream diffs below. -======= Shared internal packages under `backend/pkg/` (`gcshelper`, `oidchelper`) are also additions, not modifications, and are not tracked here. ->>>>>>> main + `jira_snowflake/tasks/convert_*.go` are adapted copies of `jira/tasks/` convertors — see the [jira_snowflake AGENTS.md](../backend/plugins/jira_snowflake/AGENTS.md) for the diff details.