diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index 3e4d47925..000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,1183 +0,0 @@
-# Copyright The Linux Foundation and each contributor to CommunityBridge.
-# SPDX-License-Identifier: MIT
-version: 2.1
-
-environment:
- BASH_ENV: ~/.bashrc
-
-setup_aws: &setup_aws
- run:
- name: Setup AWS key and Profile
- command: |
- touch ${BASH_ENV}
- if ! grep -q AWS_ACCESS_KEY_ID ${BASH_ENV} ; then
- echo "export AWS_ACCESS_KEY_ID='${!AWS_ACCESS_KEY_ID_ENV_VAR}'" >> ${BASH_ENV}
- echo "Added AWS_ACCESS_KEY_ID to ${BASH_ENV}"
- else
- echo "Skipped adding AWS_ACCESS_KEY_ID to ${BASH_ENV} - already there"
- fi
-
- if ! grep -q AWS_SECRET_ACCESS_KEY ${BASH_ENV} ; then
- echo "export AWS_SECRET_ACCESS_KEY='${!AWS_SECRET_ACCESS_KEY_ENV_VAR}'" >> ${BASH_ENV}
- echo "Added AWS_SECRET_ACCESS_KEY to ${BASH_ENV}"
- else
- echo "Skipped adding AWS_SECRET_ACCESS_KEY to ${BASH_ENV} - already there"
- fi
-
- echo "Installing Profile '${AWS_PROFILE}'..."
- mkdir -p ~/.aws
-
- touch ~/.aws/config
- if ! grep -q AWS_PROFILE ~/.aws/config; then
- printf "[profile ${AWS_PROFILE}]\nregion=${AWS_REGION}\noutput=json" > ~/.aws/config
- echo "Added ${AWS_PROFILE} profile to ~/.aws/config"
- else
- echo "Skipped adding ${AWS_PROFILE} to ~/.aws/config - already there"
- fi
-
- touch ~/.aws/credentials
- if ! grep -q AWS_PROFILE ~/.aws/credentials; then
- printf "[${AWS_PROFILE}]\naws_access_key_id=${!AWS_ACCESS_KEY_ID_ENV_VAR}\naws_secret_access_key=${!AWS_SECRET_ACCESS_KEY_ENV_VAR}" > ~/.aws/credentials
- echo "Added ${AWS_PROFILE} profile to ~/.aws/credentials"
- else
- echo "Skipped adding ${AWS_PROFILE} to ~/.aws/credentials - already there"
- fi
-
- if ! grep -q AWS_PROFILE ${BASH_ENV}; then
- echo "export AWS_PROFILE=${AWS_PROFILE}" >> ${BASH_ENV}
- echo "Added ${AWS_PROFILE} profile to ${BASH_ENV}"
- else
- echo "Skipped adding ${AWS_PROFILE} to ${BASH_ENV} - already there"
- fi
-
-install_aws_cli: &install_aws_cli
- run:
- name: Install AWS CLI Tools
- command: |
- sudo apt-get update
- sudo apt-get install -y awscli
-
-set_functional_test_environment: &set_functional_test_environment
- run:
- name: set deployment environment
- command: |
- cd && echo "Setting environment in $BASH_ENV for stage ${STAGE}" && touch $BASH_ENV
-
- # Note, we place single quotes around the values to ensure any values
- # with dollar signs are not intrepreted and expanded by accident
- # Default Test User (functional test)
- echo "export AUTH0_USERNAME='${!AUTH0_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_PASSWORD='${!AUTH0_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_CLIENT_ID='${!AUTH0_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
- # Prospective CLA Manager User (for functional tests)
- echo "export AUTH0_USER1_EMAIL='${!AUTH0_USER1_EMAIL_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER1_USERNAME='${!AUTH0_USER1_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER1_PASSWORD='${!AUTH0_USER1_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER1_CLIENT_ID='${!AUTH0_USER1_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
- # CLA Manager User (for functional tests)
- echo "export AUTH0_USER2_EMAIL='${!AUTH0_USER2_EMAIL_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER2_USERNAME='${!AUTH0_USER2_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER2_PASSWORD='${!AUTH0_USER2_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER2_CLIENT_ID='${!AUTH0_USER2_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
- # CLA Manager Intel (for functional tests)
- echo "export AUTH0_USER3_EMAIL='${!AUTH0_USER3_EMAIL_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER3_USERNAME='${!AUTH0_USER3_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER3_PASSWORD='${!AUTH0_USER3_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER3_CLIENT_ID='${!AUTH0_USER3_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
- # CLA Manager AT&T (for functional tests)
- echo "export AUTH0_USER4_EMAIL='${!AUTH0_USER4_EMAIL_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER4_USERNAME='${!AUTH0_USER4_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER4_PASSWORD='${!AUTH0_USER4_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER4_CLIENT_ID='${!AUTH0_USER4_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
- # Project Manager (for functional tests)
- echo "export AUTH0_USER5_EMAIL='${!AUTH0_USER5_EMAIL_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER5_USERNAME='${!AUTH0_USER5_USERNAME_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER5_PASSWORD='${!AUTH0_USER5_PASSWORD_ENV_VAR}'" >> ${BASH_ENV}
- echo "export AUTH0_USER5_CLIENT_ID='${!AUTH0_USER5_CLIENT_ID_ENV_VAR}'" >> ${BASH_ENV}
-
-step-library:
- - &install-node-8
- run:
- name: Install node 8
- command: |
- set +e
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash
- [ -s "${NVM_DIR}/nvm.sh" ] && \. "${NVM_DIR}/nvm.sh"
- node_version="v8.17.0"
- echo "Installing node ${node_version}..."
- nvm install ${node_version}
- nvm alias default ${node_version}
- echo "[ -s \"${NVM_DIR}/nvm.sh\" ] && . \"${NVM_DIR}/nvm.sh\"" >> $BASH_ENV
-
- - &install-node-12
- run:
- name: Install node 12
- command: |
- set +e
- curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.2/install.sh | bash
- [ -s "${NVM_DIR}/nvm.sh" ] && \. "${NVM_DIR}/nvm.sh"
- node_version="v12.20.0"
- echo "Installing node ${node_version}..."
- nvm install ${node_version}
- nvm alias default ${node_version}
- echo "[ -s \"${NVM_DIR}/nvm.sh\" ] && . \"${NVM_DIR}/nvm.sh\"" >> $BASH_ENV
-
-jobs:
- # Builds
- buildBackend: &buildBackendAnchor
- docker:
- - image: circleci/python:3.7.9-node
- steps:
- - checkout
- - add_ssh_keys:
- fingerprints:
- - "e9:13:85:f1:b1:a1:25:bf:f5:44:34:66:82:1e:31:59"
- - *setup_aws
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run:
- name: Install Top Level Dependencies
- command: |
- echo "Node version is: $(node --version)"
- echo "Running top level install..."
- yarn install
- - *install_aws_cli
- - run:
- name: Setup Backend
- command: |
- cd cla-backend
- yarn install
- echo "Upgrading pip..."
- python3 -m pip install --upgrade pip
- sudo pip install -r requirements.txt
- - run:
- name: lint
- command: |
- cd cla-backend
- ./check-headers.sh
- # Lint will always pass for now - need to continue addressing lint issues
- pylint cla/*.py || true
- - run:
- name: test
- command: |
- cd cla-backend
- export GITHUB_OAUTH_TOKEN=${GITHUB_OAUTH_TOKEN}
- pytest "cla/tests" -p no:warnings --cov="cla"
-
- buildBackendDev:
- <<: *buildBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
-
- buildBackendStaging:
- <<: *buildBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
-
- buildBackendProd:
- <<: *buildBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
-
- buildGoBackend: &buildGoBackendAnchor
- docker:
- - image: circleci/golang:1.15.6
- working_directory: /go/src/github.com/communitybridge/easycla/
- steps:
- - checkout
- - add_ssh_keys:
- fingerprints:
- - "e9:13:85:f1:b1:a1:25:bf:f5:44:34:66:82:1e:31:59"
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run: echo 'export GO111MODULE=on' >> $BASH_ENV
- - run:
- name: Setup
- command: |
- source ${BASH_ENV}
- echo "Installing python..."
- sudo apt-get update
- sudo apt install -y software-properties-common
- sudo add-apt-repository ppa:deadsnakes/ppa -y
- sudo apt-get install -y python3.7 python3-pip
- echo "Upgrading pip..."
- python3 -m pip install --upgrade pip
- echo "Python 3 version:"
- python3 --version
- make -f cla-backend-go/Makefile setup-dev
- - run:
- name: Clean
- command: |
- cd cla-backend-go
- make clean
- - run:
- name: Dependencies
- command: |
- cd cla-backend-go
- make deps
- - run:
- name: Build Swagger
- command: |
- cd cla-backend-go
- make swagger
- - run:
- name: Build
- command: |
- cd cla-backend-go
- echo "Building AWS Lambda - API..."
- make build-aws-lambda-linux
- echo "Building AWS Metrics Lambda..."
- make build-metrics-lambda-linux
- echo "Building AWS Metrics Report Lambda..."
- make build-metrics-report-lambda
- echo "Building AWS Lambda - DynamoDB Events Handler..."
- make build-dynamo-events-lambda-linux
- echo "Building AWS Lambda - Zip Builder Scheduler..."
- make build-zipbuilder-scheduler-lambda-linux
- echo "Building AWS Lambda - Zip Builder Handler..."
- make build-zipbuilder-lambda-linux
- echo "Building Functional Tests..."
- make build-functional-tests-linux
- echo "Building User Subscribe..."
- make build-user-subscribe-lambda-linux
- - run:
- name: Test
- command: |
- cd cla-backend-go
- make test
- - run:
- name: Lint
- command: |
- cd cla-backend-go
- make lint
- - run:
- name: Move Binary
- command: |
- mv cla-backend-go ~/cla-backend-go
- - persist_to_workspace:
- root: ~/
- paths:
- - cla-backend-go/backend-aws-lambda
- - cla-backend-go/user-subscribe-lambda
- - cla-backend-go/metrics-aws-lambda
- - cla-backend-go/metrics-report-lambda
- - cla-backend-go/dynamo-events-lambda
- - cla-backend-go/zipbuilder-scheduler-lambda
- - cla-backend-go/zipbuilder-lambda
- - cla-backend-go/functional-tests
-
- buildGoBackendDev:
- <<: *buildGoBackendAnchor
- environment:
- STAGE: dev
-
- buildGoBackendStaging:
- <<: *buildGoBackendAnchor
- environment:
- STAGE: staging
-
- buildGoBackendProd:
- <<: *buildGoBackendAnchor
- environment:
- STAGE: prod
-
- # Deploys
- deployBackend: &deployBackendAnchor
- docker:
- - image: circleci/python:3.7.9-node
- steps:
- - attach_workspace:
- at: ~/
- - checkout
- - add_ssh_keys:
- fingerprints:
- - "e9:13:85:f1:b1:a1:25:bf:f5:44:34:66:82:1e:31:59"
- - *setup_aws
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run:
- name: Install Top Level Dependencies
- command: |
- echo "Node version is: $(node --version)"
- echo "Running top level install..."
- yarn install
- - run:
- name: Deploy EasyCLA v1
- command: |
- echo "Using AWS profile: ${AWS_PROFILE}"
- echo "Stage is: ${STAGE}"
-
- # --------------------------------------------------------------
- ## Debug to confirm the binary files were restored
- echo "Directory: ~/"
- ls -alF ~/
- echo "Directory: ~/cla-backend-go/"
- ls -alF ~/cla-backend-go/
- ## End Debug
- # --------------------------------------------------------------
-
- # Copy over the go backend binary to the common cla-backend folder (they share a single serverless.yml config)
- cp ~/cla-backend-go/backend-aws-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/user-subscribe-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/metrics-aws-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/metrics-report-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/dynamo-events-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/zipbuilder-scheduler-lambda ~/project/cla-backend/
- cp ~/cla-backend-go/zipbuilder-lambda ~/project/cla-backend/
-
- ls -alF ~/project/cla-backend/
- pushd ~/project/cla-backend
- echo "Directory: $(pwd)"
- yarn install
-
- if [[ ! -f backend-aws-lambda ]]; then echo "Missing backend-aws-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f user-subscribe-lambda ]]; then echo "Missing user-subscribe-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f metrics-aws-lambda ]]; then echo "Missing metrics-aws-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f metrics-report-lambda ]]; then echo "Missing metrics-report-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f dynamo-events-lambda ]]; then echo "Missing dynamo-events-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f zipbuilder-lambda ]]; then echo "Missing zipbuilder-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f zipbuilder-scheduler-lambda ]]; then echo "Missing zipbuilder-scheduler-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f serverless.yml ]]; then echo "Missing serverless.yml file. Exiting..."; exit 1; fi
- if [[ ! -f serverless-authorizer.yml ]]; then echo "Missing serverless-authorizer.yml file. Exiting..."; exit 1; fi
- yarn sls deploy --force --stage ${STAGE} --region us-east-1
- - run:
- name: Deploy EasyCLA v2
- command: |
- echo "Using AWS profile: ${AWS_PROFILE}"
- echo "Stage is: ${STAGE}"
-
- # --------------------------------------------------------------
- ## Debug to confirm the binary files were restored
- echo "Directory: ~/"
- ls -alF ~/
- echo "Directory: ~/cla-backend-go/"
- ls -alF ~/cla-backend-go/
- ## End Debug
- # --------------------------------------------------------------
-
- cp ~/cla-backend-go/backend-aws-lambda ~/project/cla-backend-go/
- cp ~/cla-backend-go/user-subscribe-lambda ~/project/cla-backend-go/
- echo "Directory: ~/project/cla-backend-go/"
- ls -alF ~/project/cla-backend-go/
- pushd ~/project/cla-backend-go
- echo "Directory: $(pwd)"
- if [[ ! -f backend-aws-lambda ]]; then echo "Missing backend-aws-lambda binary file. Exiting..."; exit 1; fi
- if [[ ! -f user-subscribe-lambda ]]; then echo "Missing user-subscribe-lambda binary file. Exiting..."; exit 1; fi
- yarn install
-
- # Deploy to us-east-2
- if [[ ! -f serverless.yml ]]; then echo "Missing serverless.yml file in $(pwd). Exiting..."; exit 1; fi
- yarn sls deploy --force --stage ${STAGE} --region us-east-2
- - run:
- name: Service Check
- command: |
- sudo apt-get install -y curl
- v2_url=''
- v3_url=''
- v4_url=''
- if [[ "${STAGE}" == "prod" ]]; then
- v2_url=https://api.easycla.lfx.linuxfoundation.org/v2/health
- v3_url=https://api.easycla.lfx.linuxfoundation.org/v3/ops/health
- v4_url=https://api-gw.platform.linuxfoundation.org/cla-service/v4/ops/health
- else
- v2_url=https://api.lfcla.${STAGE}.platform.linuxfoundation.org/v2/health
- v3_url=https://api.lfcla.${STAGE}.platform.linuxfoundation.org/v3/ops/health
- v4_url=https://api-gw.${STAGE}.platform.linuxfoundation.org/cla-service/v4/ops/health
- fi
-
- echo "Validating v2 backend using endpoint: ${v2_url}"
- curl --fail -XGET ${v2_url}
- exit_code=$?
- if [[ ${exit_coe} -eq 0 ]]; then
- echo "Successful response from endpoint: ${v2_url}"
- else
- echo "Failed to get a successful response from endpoint: ${v2_url}"
- exit ${exit_code}
- fi
-
- echo "Validating v3 backend using endpoint: ${v3_url}"
- curl --fail -XGET ${v3_url}
- exit_code=$?
- if [[ ${exit_coe} -eq 0 ]]; then
- echo "Successful response from endpoint: ${v3_url}"
- # JSON response should include "Status": "healthy"
- if [[ `curl -s -XGET ${v3_url} | jq -r '.Status'` == "healthy" ]]; then
- echo "Service is healthy"
- else
- echo "Service is NOT healthy"
- exit -1
- fi
- else
- echo "Failed to get a successful response from endpoint: ${v3_url}"
- exit ${exit_code}
- fi
-
- echo "Validating v4 backend using endpoint: ${v4_url}"
- curl --fail -XGET ${v4_url}
- exit_code=$?
- if [[ ${exit_coe} -eq 0 ]]; then
- echo "Successful response from endpoint: ${v4_url}"
- # JSON response should include "Status": "healthy"
- if [[ `curl -s -XGET ${v4_url} | jq -r '.Status'` == "healthy" ]]; then
- echo "Service is healthy"
- else
- echo "Service is NOT healthy"
- exit -1
- fi
- else
- echo "Failed to get a successful response from endpoint: ${v4_url}"
- exit ${exit_code}
- fi
-
- deployBackendDev:
- <<: *deployBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
-
- deployBackendStaging:
- <<: *deployBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
- ROOT_DOMAIN: lfcla.staging.platform.linuxfoundation.org
- PRODUCT_DOMAIN: staging.lfcla.com
-
- deployBackendProd:
- <<: *deployBackendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
- ROOT_DOMAIN: lfcla.platform.linuxfoundation.org
- PRODUCT_DOMAIN: lfcla.com
-
- buildFrontend: &buildFrontendAnchor
- docker:
- - image: circleci/node:8-browsers
- steps:
- - checkout
- - *setup_aws
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run:
- name: Install Top Level Dependencies
- command: |
- echo "Node version is: $(node --version)"
- echo "Running top level install..."
- yarn install
- - *install-node-8
- - run:
- name: Install UI Dependencies
- command: |
- pushd $PROJECT_DIR
- echo "Running yarn install in folder: `pwd`. This will run yarn install in several places - see output below."
- yarn install-frontend
- popd
- - run:
- name: Build UI Source
- command: |
- echo "Building src..."
- pushd $PROJECT_DIR/src
- echo "AWS_PROFILE=${AWS_PROFILE}"
- echo "AWS_REGION=${AWS_REGION}"
- ls ~/.aws
- cat ${BASH_ENV}
- yarn prebuild:${STAGE}
- yarn build
- popd
- - run:
- name: Build Edge Source
- command: |
- echo "Building edge..."
- pushd $PROJECT_DIR/edge
- yarn build
- popd
-
- # Build Project Management Console
- buildProjectConsoleDev:
- <<: *buildFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-project-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
-
- # Build Corporate Console
- buildCorporateConsoleDev:
- <<: *buildFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-corporate-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
-
- # Build Contributor Console
- buildContributorConsoleDev:
- <<: *buildFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-contributor-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
-
- deployFrontend: &deployFrontendAnchor
- docker:
- - image: circleci/node:8-browsers
- steps:
- - checkout
- - *setup_aws
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run:
- name: Install Top Level Dependencies
- command: |
- echo "Node version is: $(node --version)"
- echo "Running top level install..."
- yarn install
- - *install-node-8
- - run:
- name: Install UI Dependencies
- command: |
- pushd $PROJECT_DIR
- echo "Running yarn install in folder: `pwd`. This will run yarn install in several places - see output below."
- yarn install-frontend
- popd
- - run:
- name: Build UI Source
- command: |
- echo "Building src..."
- pushd $PROJECT_DIR/src
- echo "Current directory is: `pwd`"
- echo "Running pre-fetch config: 'yarn prebuild:${STAGE}'..."
- yarn prebuild:${STAGE}
- echo "Running build: 'yarn build:${STAGE}'..."
- yarn build:${STAGE}
- popd
- - run:
- name: Build Edge Source
- command: |
- echo "Building edge..."
- pushd $PROJECT_DIR/edge
- echo "Current directory is: `pwd`"
- echo "Running build: 'yarn build'..."
- yarn build
- popd
- - *install-node-12
- - *install_aws_cli
-# - run:
-# name: Publish Console to S3
-# command: |
-# pushd ${PROJECT_DIR}/src
-# aws s3 rm s3://${BUCKET_NAME} --recursive
-# aws s3 sync www/ s3://${BUCKET_NAME}/ --acl public-read
-# popd
- - run:
- name: Deploy Cloudfront and LambdaEdge
- command: |
- pushd $PROJECT_DIR
- echo "Running install 'yarn install'..."
- yarn install
- echo ""
- echo "Running: yarn sls deploy --stage=\"${STAGE}\" --cloudfront=true"
- yarn sls deploy --stage="${STAGE}" --cloudfront="true"
- popd
- - run:
- name: Deploy Frontend Bucket
- command: |
- pushd $PROJECT_DIR
- echo "Running install 'yarn install'..."
- yarn install
- echo ""
- echo "Running: yarn sls client deploy --stage=\"${STAGE}\" --cloudfront=true --no-confirm --no-policy-change --no-config-change"
- yarn sls client deploy --stage="${STAGE}" --cloudfront="true" --no-confirm --no-policy-change --no-config-change
- popd
- - run:
- name: Invalidate Cache
- command: |
- pushd $PROJECT_DIR
- echo "Running: yarn sls cloudfrontInvalidate --stage=\"${STAGE}\" --region=\"${AWS_REGION}\" --cloudfront=\"true\""
- yarn sls cloudfrontInvalidate --stage="${STAGE}" --region="${AWS_REGION}" --cloudfront="true"
- popd
-
- # Project Management Console
- deployProjectManagementConsoleDev:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-project-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
- BUCKET_NAME: lf-cla-dev-cla-frontend-pmc-4
-
- deployProjectManagementConsoleStaging:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
- PROJECT_DIR: cla-frontend-project-console
- ROOT_DOMAIN: lfcla.staging.platform.linuxfoundation.org
- PRODUCT_DOMAIN: staging.lfcla.com
- BUCKET_NAME: lf-cla-staging-cla-frontend-pmc-3
-
- deployProjectManagementConsoleProd:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
- PROJECT_DIR: cla-frontend-project-console
- ROOT_DOMAIN: lfcla.platform.linuxfoundation.org
- PRODUCT_DOMAIN: lfcla.com
- BUCKET_NAME: lf-cla-prod-cla-frontend-pmc-3
-
- # Corporate Console
- deployCorporateConsoleDev:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-corporate-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
- BUCKET_NAME: lf-cla-dev-cla-frontend-cc-4
-
- deployCorporateConsoleStaging:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
- PROJECT_DIR: cla-frontend-corporate-console
- ROOT_DOMAIN: lfcla.staging.platform.linuxfoundation.org
- PRODUCT_DOMAIN: staging.lfcla.com
- BUCKET_NAME: lf-cla-staging-cla-frontend-cc-3
-
- deployCorporateConsoleProd:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
- PROJECT_DIR: cla-frontend-corporate-console
- ROOT_DOMAIN: lfcla.platform.linuxfoundation.org
- PRODUCT_DOMAIN: lfcla.com
- BUCKET_NAME: lf-cla-prod-cla-frontend-cc-3
-
- # Contributor Console
- deployContributorConsoleDev:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-frontend-contributor-console
- ROOT_DOMAIN: lfcla.dev.platform.linuxfoundation.org
- PRODUCT_DOMAIN: dev.lfcla.com
- BUCKET_NAME: lf-cla-dev-cla-frontend-ic-4
-
- deployContributorConsoleStaging:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
- PROJECT_DIR: cla-frontend-contributor-console
- ROOT_DOMAIN: lfcla.staging.platform.linuxfoundation.org
- PRODUCT_DOMAIN: staging.lfcla.com
- BUCKET_NAME: lf-cla-staging-cla-frontend-ic-3
-
- deployContributorConsoleProd:
- <<: *deployFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
- PROJECT_DIR: cla-frontend-contributor-console
- ROOT_DOMAIN: lfcla.platform.linuxfoundation.org
- PRODUCT_DOMAIN: lfcla.com
- BUCKET_NAME: lf-cla-prod-cla-frontend-ic-3
-
- deployLandingFrontend: &deployLandingFrontendAnchor
- docker:
- - image: circleci/node:8-browsers
- steps:
- - checkout
- - *setup_aws
- - run: echo 'export NVM_DIR=${HOME}/.nvm' >> $BASH_ENV
- - *install-node-12
- - run:
- name: Install Top Level Dependencies
- command: |
- echo "Node version is: $(node --version)"
- echo "Running top level install..."
- yarn install
- - run:
- name: Deploy
- command: |
- echo "Node version is: $(node --version)"
- echo "Using AWS profile: ${AWS_PROFILE}"
- echo "Stage is: ${STAGE}"
- echo "PROJECT_DIR=${PROJECT_DIR}"
-
- # Run the deploy scripts
- pushd ${PROJECT_DIR}
- echo "Current directory is: `pwd`"
-
- echo "Running install 'yarn install'..."
- yarn install
- echo "Running pre-fetch config: 'yarn prebuild:${STAGE}'..."
- yarn prebuild:${STAGE}
- echo "Running build..."
- yarn build
-
- echo "Running deploy in folder: `pwd`"
- SLS_DEBUG=* ../node_modules/serverless/bin/serverless.js deploy -s ${STAGE} -r ${AWS_REGION} --verbose
-
- echo "Running client deploy in folder: `pwd`"
- SLS_DEBUG=* ../node_modules/serverless/bin/serverless.js client deploy -s ${STAGE} -r ${AWS_REGION} --cloudfront=true --no-confirm --no-policy-change --no-config-change
-
- echo "Invalidating Cloudfront caches in folder: `pwd`"
- SLS_DEBUG=* ../node_modules/serverless/bin/serverless.js cloudfrontInvalidate -s ${STAGE} -r ${AWS_REGION} --cloudfront=true
- popd
-
- no_output_timeout: 1.5h
-
- # Landing Page
- deployLandingFrontendDev:
- <<: *deployLandingFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_DEV
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_DEV
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: dev
- PROJECT_DIR: cla-landing-page
- PRODUCT_DOMAIN: dev.lfcla.com
-
- deployLandingFrontendStaging:
- <<: *deployLandingFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_STAGING
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_STAGING
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: staging
- PROJECT_DIR: cla-landing-page
- PRODUCT_DOMAIN: staging.lfcla.com
-
- deployLandingFrontendProd:
- <<: *deployLandingFrontendAnchor
- environment:
- AWS_ACCESS_KEY_ID_ENV_VAR: AWS_ACCESS_KEY_ID_PROD
- AWS_SECRET_ACCESS_KEY_ENV_VAR: AWS_SECRET_ACCESS_KEY_PROD
- AWS_PROFILE: lf-cla
- AWS_REGION: us-east-1
- STAGE: prod
- PROJECT_DIR: cla-landing-page
- PRODUCT_DOMAIN: lfcla.com
-
- functionalTestsTavern: &functionalTestsTavern
- docker:
- - image: circleci/python:3.7.4-node
- steps:
- - attach_workspace:
- at: ~/
- - checkout
- - run:
- name: setup
- command: |
- cd tests/rest
- sudo pip3 install -r requirements.freeze.txt
- echo "Installing curl and jq..."
- sudo apt-get install -y curl jq
- - *set_functional_test_environment
- - run:
- name: functional-tests
- halt_build_on_fail: false # for now, we will pass all functional tests
- command: |
- source ${BASH_ENV}
- echo "Running functional tests for stage: ${STAGE}"
- cd tests/rest
- tavern-ci test_*.tavern.yaml --alluredir=allure_result_folder -v || true
-
- functionalTestsGo: &functionalTestsGo
- docker:
- - image: circleci/golang:1.15.6
- steps:
- - attach_workspace:
- at: ~/
- - checkout
- - *set_functional_test_environment
- - run:
- name: functional-tests-go
- command: |
- source "${BASH_ENV}"
- echo "Running golang functional tests for stage: ${STAGE}"
- echo "Home directory : $(ls ${HOME})"
- echo "${HOME}/cla-backend-go directory: $(ls ${HOME}/cla-backend-go)"
- ${HOME}/cla-backend-go/functional-tests
-
- functionalTestsTavernDev:
- <<: *functionalTestsTavern
- environment:
- # Default Functional Test User
- AUTH0_USERNAME_ENV_VAR: AUTH0_USERNAME_DEV
- AUTH0_PASSWORD_ENV_VAR: AUTH0_PASSWORD_DEV
- AUTH0_CLIENT_ID_ENV_VAR: AUTH0_CLIENT_ID_DEV
- # Prospective CLA Manager User
- AUTH0_USER1_EMAIL_ENV_VAR: AUTH0_USER1_EMAIL_DEV
- AUTH0_USER1_USERNAME_ENV_VAR: AUTH0_USER1_USERNAME_DEV
- AUTH0_USER1_PASSWORD_ENV_VAR: AUTH0_USER1_PASSWORD_DEV
- AUTH0_USER1_CLIENT_ID_ENV_VAR: AUTH0_USER1_CLIENT_ID_DEV
- # CLA Manager User
- AUTH0_USER2_EMAIL_ENV_VAR: AUTH0_USER2_EMAIL_DEV
- AUTH0_USER2_USERNAME_ENV_VAR: AUTH0_USER2_USERNAME_DEV
- AUTH0_USER2_PASSWORD_ENV_VAR: AUTH0_USER2_PASSWORD_DEV
- AUTH0_USER2_CLIENT_ID_ENV_VAR: AUTH0_USER2_CLIENT_ID_DEV
- # CLA Manager Intel
- AUTH0_USER3_EMAIL_ENV_VAR: AUTH0_USER3_EMAIL_DEV
- AUTH0_USER3_USERNAME_ENV_VAR: AUTH0_USER3_USERNAME_DEV
- AUTH0_USER3_PASSWORD_ENV_VAR: AUTH0_USER3_PASSWORD_DEV
- AUTH0_USER3_CLIENT_ID_ENV_VAR: AUTH0_USER3_CLIENT_ID_DEV
- # CLA Manager AT&T
- AUTH0_USER4_EMAIL_ENV_VAR: AUTH0_USER4_EMAIL_DEV
- AUTH0_USER4_USERNAME_ENV_VAR: AUTH0_USER4_USERNAME_DEV
- AUTH0_USER4_PASSWORD_ENV_VAR: AUTH0_USER4_PASSWORD_DEV
- AUTH0_USER4_CLIENT_ID_ENV_VAR: AUTH0_USER4_CLIENT_ID_DEV
- # Project Manager ColorIO
- AUTH0_USER5_EMAIL_ENV_VAR: AUTH0_USER5_EMAIL_DEV
- AUTH0_USER5_USERNAME_ENV_VAR: AUTH0_USER5_USERNAME_DEV
- AUTH0_USER5_PASSWORD_ENV_VAR: AUTH0_USER5_PASSWORD_DEV
- AUTH0_USER5_CLIENT_ID_ENV_VAR: AUTH0_USER5_CLIENT_ID_DEV
- API_URL: 'https://api.lfcla.dev.platform.linuxfoundation.org'
- V2_API_URL: 'https://api-gw.dev.platform.linuxfoundation.org/cla-service'
- STAGE: dev
-
- functionalTestsGoDev:
- <<: *functionalTestsGo
- environment:
- # Default Functional Test User
- AUTH0_USERNAME_ENV_VAR: AUTH0_USERNAME_DEV
- AUTH0_PASSWORD_ENV_VAR: AUTH0_PASSWORD_DEV
- AUTH0_CLIENT_ID_ENV_VAR: AUTH0_CLIENT_ID_DEV
- # Prospective CLA Manager User
- AUTH0_USER1_EMAIL_ENV_VAR: AUTH0_USER1_EMAIL_DEV
- AUTH0_USER1_USERNAME_ENV_VAR: AUTH0_USER1_USERNAME_DEV
- AUTH0_USER1_PASSWORD_ENV_VAR: AUTH0_USER1_PASSWORD_DEV
- AUTH0_USER1_CLIENT_ID_ENV_VAR: AUTH0_USER1_CLIENT_ID_DEV
- # CLA Manager User
- AUTH0_USER2_EMAIL_ENV_VAR: AUTH0_USER2_EMAIL_DEV
- AUTH0_USER2_USERNAME_ENV_VAR: AUTH0_USER2_USERNAME_DEV
- AUTH0_USER2_PASSWORD_ENV_VAR: AUTH0_USER2_PASSWORD_DEV
- AUTH0_USER2_CLIENT_ID_ENV_VAR: AUTH0_USER2_CLIENT_ID_DEV
- # CLA Manager Intel
- AUTH0_USER3_EMAIL_ENV_VAR: AUTH0_USER3_EMAIL_DEV
- AUTH0_USER3_USERNAME_ENV_VAR: AUTH0_USER3_USERNAME_DEV
- AUTH0_USER3_PASSWORD_ENV_VAR: AUTH0_USER3_PASSWORD_DEV
- AUTH0_USER3_CLIENT_ID_ENV_VAR: AUTH0_USER3_CLIENT_ID_DEV
- # CLA Manager AT&T
- AUTH0_USER4_EMAIL_ENV_VAR: AUTH0_USER4_EMAIL_DEV
- AUTH0_USER4_USERNAME_ENV_VAR: AUTH0_USER4_USERNAME_DEV
- AUTH0_USER4_PASSWORD_ENV_VAR: AUTH0_USER4_PASSWORD_DEV
- AUTH0_USER4_CLIENT_ID_ENV_VAR: AUTH0_USER4_CLIENT_ID_DEV
- # Project Manager ColorIO
- AUTH0_USER5_EMAIL_ENV_VAR: AUTH0_USER5_EMAIL_DEV
- AUTH0_USER5_USERNAME_ENV_VAR: AUTH0_USER5_USERNAME_DEV
- AUTH0_USER5_PASSWORD_ENV_VAR: AUTH0_USER5_PASSWORD_DEV
- AUTH0_USER5_CLIENT_ID_ENV_VAR: AUTH0_USER5_CLIENT_ID_DEV
- API_URL: 'https://api.lfcla.dev.platform.linuxfoundation.org'
- V2_API_URL: 'https://api-gw.dev.platform.linuxfoundation.org/cla-service'
- STAGE: dev
-
-workflows:
- version: 2.1
- build_and_deploy:
- jobs:
- - buildBackendDev:
- filters:
- tags:
- only: /.*/
- - buildGoBackendDev:
- filters:
- tags:
- only: /.*/
- - buildProjectConsoleDev:
- filters:
- tags:
- only: /.*/
- - buildCorporateConsoleDev:
- filters:
- tags:
- only: /.*/
- - buildContributorConsoleDev:
- filters:
- tags:
- only: /.*/
-
- # Deploy Dev
- - deployBackendDev:
- requires:
- - buildBackendDev
- - buildGoBackendDev
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - deployProjectManagementConsoleDev:
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - deployCorporateConsoleDev:
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - deployContributorConsoleDev:
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - deployLandingFrontendDev:
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - functionalTestsGoDev:
- requires:
- - deployBackendDev
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
- - functionalTestsTavernDev:
- requires:
- - deployBackendDev
- filters:
- tags:
- ignore: /.*/
- branches:
- only:
- - main
-
- # Deploy Staging
- - buildBackendStaging:
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - buildGoBackendStaging:
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - approve_staging:
- type: approval
- requires:
- - buildBackendStaging
- - buildGoBackendStaging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployBackendStaging:
- requires:
- - approve_staging
- - buildBackendStaging
- - buildGoBackendStaging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployProjectManagementConsoleStaging:
- requires:
- - approve_staging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployCorporateConsoleStaging:
- requires:
- - approve_staging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployContributorConsoleStaging:
- requires:
- - approve_staging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployLandingFrontendStaging:
- requires:
- - approve_staging
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
-
- # Deploy Prod
- - buildBackendProd:
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - buildGoBackendProd:
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - approve_prod:
- type: approval
- requires:
- - buildBackendProd
- - buildGoBackendProd
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployBackendProd:
- requires:
- - approve_prod
- - buildBackendProd
- - buildGoBackendProd
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployProjectManagementConsoleProd:
- requires:
- - approve_prod
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployCorporateConsoleProd:
- requires:
- - approve_prod
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployContributorConsoleProd:
- requires:
- - approve_prod
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
- - deployLandingFrontendProd:
- requires:
- - approve_prod
- filters:
- branches:
- ignore: /.*/
- tags:
- # see semver examples https://regex101.com/r/Ly7O1x/201/
- only: /^v?(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$/
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 3bbf4d3cd..8f020ed2c 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -32,12 +32,20 @@ If applicable, add screenshots to help explain your problem.
Please complete the following information:
* Environment:
+ - [ ] ALL
- [ ] DEV
- [ ] STAGING
- [ ] PROD
* Browser:
- [ ] Chrome/Brave
- [ ] Firefox
+ - [ ] Opera
+ - [ ] Vivaldi
+ - [ ] LibreWolf
+ - [ ] SRware Iron
+ - [ ] Dissenter
+ - [ ] Slimjet
+ - [ ] Midori
- [ ] Edge
- [ ] Lynx
* Version: v1.0.XX
diff --git a/.github/ISSUE_TEMPLATE/docs_update.md b/.github/ISSUE_TEMPLATE/docs_update.md
index 5d59ed86c..1cc4d6328 100644
--- a/.github/ISSUE_TEMPLATE/docs_update.md
+++ b/.github/ISSUE_TEMPLATE/docs_update.md
@@ -10,7 +10,7 @@ assignees: ''
Describe what has changed and which documents need updating
-* [/docs/doc-to-update.md](https://github.com/communitybridge/easycla/blob/master/docs/)
+* [/docs/doc-to-update.md](https://github.com/linuxfoundation/easycla/blob/main/docs/)
## Tasks
@@ -29,7 +29,7 @@ The "done" criteria when this feature or problem is resolved. Such as:
1. Documentation changes submitted as a Pull Request
1. Pull Request Reviewed and Approved by Product Owner
-1. Documentation changes merged to 'master' branch
+1. Documentation changes merged to 'main' branch
## Images
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..de0bdfac6
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,31 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+ - package-ecosystem: "npm" # See documentation for possible values
+ directory: "/cla-landing-page" # Location of package manifests
+ schedule:
+ interval: "weekly"
+ - package-ecosystem: "npm" # See documentation for possible values
+ directory: "/cla-backend" # Location of package manifests
+ schedule:
+ interval: "weekly"
+ - package-ecosystem: "pip" # See documentation for possible values
+ directory: "/cla-backend" # Location of package manifests
+ schedule:
+ interval: "weekly"
+ - package-ecosystem: "npm" # See documentation for possible values
+ directory: "/cla-backend-go" # Location of package manifests
+ schedule:
+ interval: "weekly"
+ - package-ecosystem: "gomod" # See documentation for possible values
+ directory: "/cla-backend-go" # Location of package manifests
+ schedule:
+ interval: "weekly"
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
new file mode 100644
index 000000000..67b9a6b71
--- /dev/null
+++ b/.github/workflows/build-pr.yml
@@ -0,0 +1,111 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Build and Test Pull Request
+on:
+ pull_request:
+ branches:
+ - dev
+
+permissions:
+ id-token: write
+ contents: read
+ pull-requests: write
+
+env:
+ AWS_REGION: us-east-1
+ STAGE: dev
+
+jobs:
+ build-test-lint:
+ runs-on: ubuntu-latest
+ environment: dev
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.24'
+ - name: Go Version
+ run: go version
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ - name: Setup python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: 'pip'
+ - name: Cache Go modules
+ uses: actions/cache@v3
+ with:
+ path: ${{ github.workspace }}/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+
+ - name: Configure Git to clone private Github repos
+ run: git config --global url."https://${TOKEN_USER}:${TOKEN}@github.com".insteadOf "https://github.com"
+ env:
+ TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }}
+ TOKEN_USER: ${{ secrets.PERSONAL_ACCESS_TOKEN_USER_GITHUB }}
+
+ - name: Add OS Tools
+ run: sudo apt update && sudo apt-get install file -y
+
+ - name: Python Setup
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pip
+ pip install -r requirements.txt
+
+ - name: Python Lint
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pylint
+ pylint cla/*.py || true
+
+ - name: Python Test
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pytest py pytest-cov pytest-clarity
+ pytest "cla/tests" -p no:warnings
+ env:
+ PLATFORM_GATEWAY_URL: https://api-gw.dev.platform.linuxfoundation.org
+ AUTH0_PLATFORM_URL: https://linuxfoundation-dev.auth0.com/oauth/token
+ AUTH0_PLATFORM_CLIENT_ID: ${{ secrets.AUTH0_PLATFORM_CLIENT_ID }}
+ AUTH0_PLATFORM_CLIENT_SECRET: ${{ secrets.AUTH0_PLATFORM_CLIENT_SECRET }}
+ AUTH0_PLATFORM_AUDIENCE: https://api-gw.dev.platform.linuxfoundation.org/
+
+ - name: Go Setup
+ working-directory: cla-backend-go
+ run: make clean setup
+
+ - name: Go Dependencies
+ working-directory: cla-backend-go
+ run: make deps
+
+ - name: Go Swagger Generate
+ working-directory: cla-backend-go
+ run: make swagger
+
+ - name: Go Build
+ working-directory: cla-backend-go
+ run: |
+ make build-lambdas-linux build-functional-tests-linux
+
+ - name: Go Test
+ working-directory: cla-backend-go
+ run: make test
+
+ - name: Go Lint
+ working-directory: cla-backend-go
+ run: make lint
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index c4d7d1cbf..8f512f781 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -1,11 +1,14 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
name: "CodeQL"
on:
push:
- branches: [master, ]
+ branches: [main, ]
pull_request:
# The branches below must be a subset of the branches above
- branches: [master]
+ branches: [main]
schedule:
- cron: '0 5 * * 4'
@@ -16,7 +19,7 @@ jobs:
steps:
- name: Checkout repository
- uses: actions/checkout@v2
+ uses: actions/checkout@v3
with:
# We must fetch at least the immediate parents so that if this is
# a pull request then we can checkout the head.
@@ -24,8 +27,9 @@ jobs:
# If this run was triggered by a pull request event, then checkout
# the head of the pull request instead of the merge commit.
- - run: git checkout HEAD^2
- if: ${{ github.event_name == 'pull_request' }}
+ # Note: git checkout HEAD^2 is no longer necessary. Please remove this step as Code Scanning recommends analyzing the merge commit for best results.
+ #- run: git checkout HEAD^2
+ # if: ${{ github.event_name == 'pull_request' }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
diff --git a/.github/workflows/cypress-functional-pr.yml b/.github/workflows/cypress-functional-pr.yml
new file mode 100644
index 000000000..a03e239b3
--- /dev/null
+++ b/.github/workflows/cypress-functional-pr.yml
@@ -0,0 +1,95 @@
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Cypress Functional Tests (PR) - runs on *currently* deployed dev API, not the new one from this PR.
+
+on:
+ pull_request:
+ branches:
+ - dev
+
+permissions:
+ contents: read
+
+jobs:
+ cypress-functional:
+ if: ${{ github.event.pull_request.head.repo.fork == false }}
+ runs-on: ubuntu-latest
+ continue-on-error: true
+ defaults:
+ run:
+ working-directory: tests/functional
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ cache: 'npm'
+
+ - name: Install system dependencies
+ shell: bash
+ run: |
+ set -euo pipefail
+ sudo apt-get update
+ # Core deps for Cypress/Electron under Xvfb
+ sudo apt-get install -y \
+ xvfb \
+ libgtk-3-0 \
+ libgbm1 \
+ libnss3 \
+ libxss1 \
+ xauth \
+ fonts-liberation \
+ xdg-utils \
+ ca-certificates \
+ libatk-bridge2.0-0 \
+ libatspi2.0-0 \
+ libdrm2
+ # Optional/legacy GTK2 (ok if missing)
+ sudo apt-get install -y libgtk2.0-0 || true
+ # Audio lib: Noble uses libasound2t64 (fallback to libasound2 on older images)
+ sudo apt-get install -y libasound2t64 || sudo apt-get install -y libasound2 || true
+ # Notify lib: prefer runtime package; fall back to -dev if needed
+ sudo apt-get install -y libnotify4 || sudo apt-get install -y libnotify-dev || true
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Create .env from secrets and constants
+ run: |
+ cat > .env <<'EOF'
+ APP_URL=https://api-gw.dev.platform.linuxfoundation.org/
+ AUTH0_TOKEN_API=https://linuxfoundation-dev.auth0.com/oauth/token
+ CYPRESS_ENV=dev
+
+ AUTH0_USER_NAME=${{ secrets.AUTH0_USER_NAME }}
+ AUTH0_PASSWORD=${{ secrets.AUTH0_PASSWORD }}
+ LFX_API_TOKEN=${{ secrets.LFX_API_TOKEN }}
+ AUTH0_CLIENT_SECRET=${{ secrets.AUTH0_CLIENT_SECRET }}
+ AUTH0_CLIENT_ID=${{ secrets.AUTH0_CLIENT_ID }}
+ EOF
+ echo "Wrote $(pwd)/.env"
+
+ - name: Show Cypress version
+ run: npx cypress --version
+
+ - name: Verify Cypress binary
+ run: npx cypress verify
+
+ - name: Run Cypress (xvfb)
+ run: xvfb-run -a npx cypress run
+
+ - name: Upload Cypress Artifacts (on failure)
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cypress-artifacts
+ path: |
+ tests/functional/cypress/screenshots
+ tests/functional/cypress/videos
+ if-no-files-found: ignore
+
diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml
new file mode 100644
index 000000000..af6d370b8
--- /dev/null
+++ b/.github/workflows/deploy-dev.yml
@@ -0,0 +1,307 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Build and Deploy to DEV
+on:
+ push:
+ branches:
+ - dev
+
+permissions:
+ # These permissions are needed to interact with GitHub's OIDC Token endpoint to fetch/set the AWS deployment credentials.
+ id-token: write
+ contents: read
+
+env:
+ AWS_REGION: us-east-1
+ STAGE: dev
+
+jobs:
+ build-deploy-dev:
+ runs-on: ubuntu-latest
+ environment: dev
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Setup go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.24'
+
+ - name: Go Version
+ run: go version
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+
+ - name: Setup python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: 'pip'
+
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ audience: sts.amazonaws.com
+ role-to-assume: arn:aws:iam::395594542180:role/github-actions-deploy
+ aws-region: us-east-1
+
+ - name: Cache Go modules
+ uses: actions/cache@v3
+ with:
+ path: ${{ github.workspace }}/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+
+ - name: Configure Git to clone private Github repos
+ run: git config --global url."https://${TOKEN_USER}:${TOKEN}@github.com".insteadOf "https://github.com"
+ env:
+ TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }}
+ TOKEN_USER: ${{ secrets.PERSONAL_ACCESS_TOKEN_USER_GITHUB }}
+
+ - name: Add OS Tools
+ run: sudo apt update && sudo apt-get install file -y
+
+ - name: Python Setup
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pip
+ pip install -r requirements.txt
+
+ - name: Python Lint
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pylint
+ pylint cla/*.py || true
+
+ - name: Python Test
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pytest py pytest-cov pytest-clarity
+ pytest "cla/tests" -p no:warnings
+ env:
+ PLATFORM_GATEWAY_URL: https://api-gw.dev.platform.linuxfoundation.org
+ AUTH0_PLATFORM_URL: https://linuxfoundation-dev.auth0.com/oauth/token
+ AUTH0_PLATFORM_CLIENT_ID: ${{ secrets.AUTH0_PLATFORM_CLIENT_ID }}
+ AUTH0_PLATFORM_CLIENT_SECRET: ${{ secrets.AUTH0_PLATFORM_CLIENT_SECRET }}
+ AUTH0_PLATFORM_AUDIENCE: https://api-gw.dev.platform.linuxfoundation.org/
+
+ - name: Go Setup
+ working-directory: cla-backend-go
+ run: |
+ make clean setup
+
+ - name: Go Dependencies
+ working-directory: cla-backend-go
+ run: make deps
+
+ - name: Go Swagger Generate
+ working-directory: cla-backend-go
+ run: |
+ make swagger
+
+ - name: Go Build
+ working-directory: cla-backend-go
+ run: |
+ make build-lambdas-linux build-functional-tests-linux
+
+ - name: Go Test
+ working-directory: cla-backend-go
+ run: make test
+
+ - name: Go Lint
+ working-directory: cla-backend-go
+ run: make lint
+
+ - name: Setup Deployment
+ working-directory: cla-backend
+ run: |
+ mkdir -p bin
+ cp ../cla-backend-go/bin/backend-aws-lambda bin/
+ cp ../cla-backend-go/bin/user-subscribe-lambda bin/
+ cp ../cla-backend-go/bin/metrics-aws-lambda bin/
+ cp ../cla-backend-go/bin/metrics-report-lambda bin/
+ cp ../cla-backend-go/bin/dynamo-events-lambda bin/
+ cp ../cla-backend-go/bin/zipbuilder-scheduler-lambda bin/
+ cp ../cla-backend-go/bin/zipbuilder-lambda bin/
+ cp ../cla-backend-go/bin/gitlab-repository-check-lambda bin/
+
+ - name: EasyCLA v1 Deployment us-east-1
+ working-directory: cla-backend
+ run: |
+ yarn install
+ if [[ ! -f bin/backend-aws-lambda ]]; then echo "Missing bin/backend-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/user-subscribe-lambda ]]; then echo "Missing bin/user-subscribe-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/metrics-aws-lambda ]]; then echo "Missing bin/metrics-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/metrics-report-lambda ]]; then echo "Missing bin/metrics-report-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/dynamo-events-lambda ]]; then echo "Missing bin/dynamo-events-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/zipbuilder-lambda ]]; then echo "Missing bin/zipbuilder-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/zipbuilder-scheduler-lambda ]]; then echo "Missing bin/zipbuilder-scheduler-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/gitlab-repository-check-lambda ]]; then echo "Missing bin/gitlab-repository-check-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f serverless.yml ]]; then echo "Missing serverless.yml file. Exiting..."; exit 1; fi
+ if [[ ! -f serverless-authorizer.yml ]]; then echo "Missing serverless-authorizer.yml file. Exiting..."; exit 1; fi
+ yarn sls deploy --force --stage ${STAGE} --region us-east-1 --verbose
+
+ - name: EasyCLA v1 Service Check
+ run: |
+ sudo apt install curl jq -y
+
+ # Development environment endpoints to test
+ declare -r v2_url="https://api.lfcla.${STAGE}.platform.linuxfoundation.org/v2/health"
+ declare -r v3_url="https://api.lfcla.${STAGE}.platform.linuxfoundation.org/v3/ops/health"
+
+ echo "Validating v2 backend using endpoint: ${v2_url}"
+ curl --fail -XGET ${v2_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v2_url}"
+ else
+ echo "Failed to get a successful response from endpoint: ${v2_url}"
+ exit ${exit_code}
+ fi
+
+ echo "Validating v3 backend using endpoint: ${v3_url}"
+ curl --fail -XGET ${v3_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v3_url}"
+ # JSON response should include "Status": "healthy"
+ if [[ `curl -s -XGET ${v3_url} | jq -r '.Status'` == "healthy" ]]; then
+ echo "Service is healthy"
+ else
+ echo "Service is NOT healthy"
+ exit -1
+ fi
+ else
+ echo "Failed to get a successful response from endpoint: ${v3_url}"
+ exit ${exit_code}
+ fi
+
+ - name: EasyCLA v2 Deployment us-east-2
+ working-directory: cla-backend-go
+ run: |
+ if [[ ! -f bin/backend-aws-lambda ]]; then echo "Missing bin/backend-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/user-subscribe-lambda ]]; then echo "Missing bin/user-subscribe-lambda binary file. Exiting..."; exit 1; fi
+ rm -rf ./node_modules/
+ yarn install
+ yarn sls deploy --force --stage ${STAGE} --region us-east-2 --verbose
+
+ - name: EasyCLA v2 Service Check
+ run: |
+ sudo apt install curl jq -y
+
+ # Development environment endpoint to test
+ v4_url="https://api-gw.${STAGE}.platform.linuxfoundation.org/cla-service/v4/ops/health"
+
+ echo "Validating v4 backend using endpoint: ${v4_url}"
+ curl --fail -XGET ${v4_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v4_url}"
+ # JSON response should include "Status": "healthy"
+ if [[ `curl -s -XGET ${v4_url} | jq -r '.Status'` == "healthy" ]]; then
+ echo "Service is healthy"
+ else
+ echo "Service is NOT healthy"
+ exit -1
+ fi
+ else
+ echo "Failed to get a successful response from endpoint: ${v4_url}"
+ exit ${exit_code}
+ fi
+
+
+ cypress-functional-after-deploy:
+ name: Cypress Functional Tests (post-deploy) - executes on a freshly deployed dev API.
+ if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false }}
+ runs-on: ubuntu-latest
+ continue-on-error: true
+ timeout-minutes: 75
+ needs: build-deploy-dev
+ environment: dev
+ defaults:
+ run:
+ working-directory: tests/functional
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v4
+
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ cache: 'npm'
+
+ - name: Install system dependencies
+ shell: bash
+ run: |
+ set -euo pipefail
+ sudo apt-get update
+ # Core deps for Cypress/Electron under Xvfb
+ sudo apt-get install -y \
+ xvfb \
+ libgtk-3-0 \
+ libgbm1 \
+ libnss3 \
+ libxss1 \
+ xauth \
+ fonts-liberation \
+ xdg-utils \
+ ca-certificates \
+ libatk-bridge2.0-0 \
+ libatspi2.0-0 \
+ libdrm2
+ # Optional/legacy GTK2 (ok if missing)
+ sudo apt-get install -y libgtk2.0-0 || true
+ # Audio lib: Noble uses libasound2t64 (fallback to libasound2 on older images)
+ sudo apt-get install -y libasound2t64 || sudo apt-get install -y libasound2 || true
+ # Notify lib: prefer runtime package; fall back to -dev if needed
+ sudo apt-get install -y libnotify4 || sudo apt-get install -y libnotify-dev || true
+
+ - name: Install dependencies
+ run: npm ci
+
+ - name: Create .env from secrets and constants
+ run: |
+ cat > .env <<'EOF'
+ APP_URL=https://api-gw.dev.platform.linuxfoundation.org/
+ AUTH0_TOKEN_API=https://linuxfoundation-dev.auth0.com/oauth/token
+ CYPRESS_ENV=dev
+
+ AUTH0_USER_NAME=${{ secrets.AUTH0_USER_NAME }}
+ AUTH0_PASSWORD=${{ secrets.AUTH0_PASSWORD }}
+ LFX_API_TOKEN=${{ secrets.LFX_API_TOKEN }}
+ AUTH0_CLIENT_SECRET=${{ secrets.AUTH0_CLIENT_SECRET }}
+ AUTH0_CLIENT_ID=${{ secrets.AUTH0_CLIENT_ID }}
+ EOF
+ echo "Wrote $(pwd)/.env"
+
+ - name: Show Cypress version
+ run: npx cypress --version
+
+ - name: Verify Cypress binary
+ run: npx cypress verify
+
+ - name: Run Cypress (xvfb)
+ run: xvfb-run -a npx cypress run
+
+ - name: Upload Cypress Artifacts (on failure)
+ if: failure()
+ uses: actions/upload-artifact@v4
+ with:
+ name: cypress-artifacts-post-deploy
+ path: |
+ tests/functional/cypress/screenshots
+ tests/functional/cypress/videos
+ if-no-files-found: ignore
diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml
new file mode 100644
index 000000000..0a48fa0e8
--- /dev/null
+++ b/.github/workflows/deploy-prod.yml
@@ -0,0 +1,187 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Build and Deploy to PROD
+
+on:
+ push:
+ tags:
+ - v1.*
+ - v2.*
+
+permissions:
+ # These permissions are needed to interact with GitHub's OIDC Token endpoint to fetch/set the AWS deployment credentials.
+ id-token: write
+ contents: read
+
+env:
+ AWS_REGION: us-east-1
+ STAGE: prod
+
+jobs:
+ build-deploy-prod:
+ runs-on: ubuntu-latest
+ environment: prod
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '1.24'
+ - name: Go Version
+ run: go version
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ - name: Setup python
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+ cache: 'pip'
+ - name: Configure AWS Credentials
+ uses: aws-actions/configure-aws-credentials@v4
+ with:
+ audience: sts.amazonaws.com
+ role-to-assume: arn:aws:iam::716487311010:role/github-actions-deploy
+ aws-region: us-east-1
+ - name: Cache Go modules
+ uses: actions/cache@v3
+ with:
+ path: ${{ github.workspace }}/go/pkg/mod
+ key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+ restore-keys: |
+ ${{ runner.os }}-go-
+
+ - name: Configure Git to clone private Github repos
+ run: git config --global url."https://${TOKEN_USER}:${TOKEN}@github.com".insteadOf "https://github.com"
+ env:
+ TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN_GITHUB }}
+ TOKEN_USER: ${{ secrets.PERSONAL_ACCESS_TOKEN_USER_GITHUB }}
+
+ - name: Add OS Tools
+ run: sudo apt update && sudo apt-get install file -y
+
+ - name: Python Setup
+ working-directory: cla-backend
+ run: |
+ python -m venv .venv
+ source .venv/bin/activate
+ pip install --upgrade pip
+ pip install -r requirements.txt
+
+ - name: Go Setup
+ working-directory: cla-backend-go
+ run: |
+ make clean setup
+
+ - name: Go Dependencies
+ working-directory: cla-backend-go
+ run: make deps
+
+ - name: Go Swagger Generate
+ working-directory: cla-backend-go
+ run: |
+ make swagger
+
+ - name: Go Build
+ working-directory: cla-backend-go
+ run: |
+ make build-lambdas-linux build-functional-tests-linux
+
+ - name: Setup Deployment
+ working-directory: cla-backend
+ run: |
+ mkdir -p bin
+ cp ../cla-backend-go/bin/backend-aws-lambda bin/
+ cp ../cla-backend-go/bin/user-subscribe-lambda bin/
+ cp ../cla-backend-go/bin/metrics-aws-lambda bin/
+ cp ../cla-backend-go/bin/metrics-report-lambda bin/
+ cp ../cla-backend-go/bin/dynamo-events-lambda bin/
+ cp ../cla-backend-go/bin/zipbuilder-scheduler-lambda bin/
+ cp ../cla-backend-go/bin/zipbuilder-lambda bin/
+ cp ../cla-backend-go/bin/gitlab-repository-check-lambda bin/
+
+ - name: EasyCLA v1 Deployment us-east-1
+ working-directory: cla-backend
+ run: |
+ yarn install
+ if [[ ! -f bin/backend-aws-lambda ]]; then echo "Missing bin/backend-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/user-subscribe-lambda ]]; then echo "Missing bin/user-subscribe-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/metrics-aws-lambda ]]; then echo "Missing bin/metrics-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/metrics-report-lambda ]]; then echo "Missing bin/metrics-report-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/dynamo-events-lambda ]]; then echo "Missing bin/dynamo-events-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/zipbuilder-lambda ]]; then echo "Missing bin/zipbuilder-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/zipbuilder-scheduler-lambda ]]; then echo "Missing bin/zipbuilder-scheduler-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/gitlab-repository-check-lambda ]]; then echo "Missing bin/gitlab-repository-check-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f serverless.yml ]]; then echo "Missing serverless.yml file. Exiting..."; exit 1; fi
+ if [[ ! -f serverless-authorizer.yml ]]; then echo "Missing serverless-authorizer.yml file. Exiting..."; exit 1; fi
+ yarn sls deploy --force --stage ${STAGE} --region us-east-1 --verbose
+
+ - name: EasyCLA v1 Service Check
+ run: |
+ sudo apt install curl jq -y
+
+ # Production environment endpoints to test
+ declare -r v2_url="https://api.easycla.lfx.linuxfoundation.org/v2/health"
+ declare -r v3_url="https://api.easycla.lfx.linuxfoundation.org/v3/ops/health"
+
+ echo "Validating v2 backend using endpoint: ${v2_url}"
+ curl --fail -XGET ${v2_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v2_url}"
+ else
+ echo "Failed to get a successful response from endpoint: ${v2_url}"
+ exit ${exit_code}
+ fi
+
+ echo "Validating v3 backend using endpoint: ${v3_url}"
+ curl --fail -XGET ${v3_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v3_url}"
+ # JSON response should include "Status": "healthy"
+ if [[ `curl -s -XGET ${v3_url} | jq -r '.Status'` == "healthy" ]]; then
+ echo "Service is healthy"
+ else
+ echo "Service is NOT healthy"
+ exit -1
+ fi
+ else
+ echo "Failed to get a successful response from endpoint: ${v3_url}"
+ exit ${exit_code}
+ fi
+ - name: EasyCLA v2 Deployment us-east-2
+ working-directory: cla-backend-go
+ run: |
+ if [[ ! -f bin/backend-aws-lambda ]]; then echo "Missing bin/backend-aws-lambda binary file. Exiting..."; exit 1; fi
+ if [[ ! -f bin/user-subscribe-lambda ]]; then echo "Missing bin/user-subscribe-lambda binary file. Exiting..."; exit 1; fi
+ rm -rf ./node_modules/
+ yarn install
+ yarn sls deploy --force --stage ${STAGE} --region us-east-2
+
+ - name: EasyCLA v2 Service Check
+ run: |
+ sudo apt install curl jq -y
+
+ # Production environment endpoint to test
+ v4_url="https://api-gw.platform.linuxfoundation.org/cla-service/v4/ops/health"
+
+ echo "Validating v4 backend using endpoint: ${v4_url}"
+ curl --fail -XGET ${v4_url}
+ exit_code=$?
+ if [[ ${exit_code} -eq 0 ]]; then
+ echo "Successful response from endpoint: ${v4_url}"
+ # JSON response should include "Status": "healthy"
+ if [[ `curl -s -XGET ${v4_url} | jq -r '.Status'` == "healthy" ]]; then
+ echo "Service is healthy"
+ else
+ echo "Service is NOT healthy"
+ exit -1
+ fi
+ else
+ echo "Failed to get a successful response from endpoint: ${v4_url}"
+ exit ${exit_code}
+ fi
diff --git a/.github/workflows/license-header-check.yml b/.github/workflows/license-header-check.yml
new file mode 100644
index 000000000..cae5515fb
--- /dev/null
+++ b/.github/workflows/license-header-check.yml
@@ -0,0 +1,31 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: License Header Check
+
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+jobs:
+ license-header-check:
+ name: License Header Check
+ runs-on: ubuntu-latest
+ environment: dev
+
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+ - name: Check License Headers - Python
+ working-directory: cla-backend
+ run: |
+ ./check-headers.sh
+ - name: Check License Headers - Go
+ working-directory: cla-backend-go
+ run: |
+ ./check-headers.sh
diff --git a/.github/workflows/yarn-scan-backend-go-pr.yml b/.github/workflows/yarn-scan-backend-go-pr.yml
new file mode 100644
index 000000000..43688e3d8
--- /dev/null
+++ b/.github/workflows/yarn-scan-backend-go-pr.yml
@@ -0,0 +1,28 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Yarn Golang Backend Dependency Audit
+
+on:
+ # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+ pull_request:
+ branches:
+ - dev
+
+jobs:
+ yarn-scan-backend-go-pr:
+ runs-on: ubuntu-latest
+ environment: dev
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ - name: Setup
+ run: yarn install
+ - name: Yarn Audit
+ working-directory: cla-backend-go
+ run: |
+ yarn audit
diff --git a/.github/workflows/yarn-scan-backend-pr.yml b/.github/workflows/yarn-scan-backend-pr.yml
new file mode 100644
index 000000000..51a2f2f2d
--- /dev/null
+++ b/.github/workflows/yarn-scan-backend-pr.yml
@@ -0,0 +1,28 @@
+---
+# Copyright The Linux Foundation and each contributor to CommunityBridge.
+# SPDX-License-Identifier: MIT
+
+name: Yarn Python Backend Dependency Audit
+
+on:
+ # https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+ pull_request:
+ branches:
+ - dev
+
+jobs:
+ yarn-scan-backend-pr:
+ runs-on: ubuntu-latest
+ environment: dev
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Node
+ uses: actions/setup-node@v4
+ with:
+ node-version: '20'
+ - name: Setup
+ run: yarn install
+ - name: Yarn Audit
+ working-directory: cla-backend
+ run: |
+ yarn audit
diff --git a/.gitignore b/.gitignore
index 311eba788..b4346d922 100755
--- a/.gitignore
+++ b/.gitignore
@@ -239,3 +239,23 @@ dist/*
.playground
api-postman/*
+
+cla-backend/run-python-test-example-*.py
+
+# LG
+out
+*.secret
+*log*.json
+cover.out
+
+# Cypress test outputs
+**/cypress/screenshots/
+**/cypress/videos/
+**/cypress/reports/
+
+# Local env vars
+.env
+src.txt
+src.txt.*
+out.json
+body.json
diff --git a/BOT_ALLOWLIST.md b/BOT_ALLOWLIST.md
new file mode 100644
index 000000000..e9ad26a3e
--- /dev/null
+++ b/BOT_ALLOWLIST.md
@@ -0,0 +1,182 @@
+## Allowlisting Bots
+
+You can allow specific bot users to automatically pass the CLA check.
+
+This can be done on the GitHub organization level by setting the `skip_cla` property on `cla-{stage}-github-orgs` DynamoDB table.
+
+Replace `{stage}` with either `dev` or `prod`.
+
+This property is a map attribute that contains mapping from repository pattern to bot GitHub login, email and name pattern.
+
+Example `login` is `lukaszgryglicki` (like any `login` that can be accessed via `https://github.com/login`).
+
+This is sometimes called `username` but we use `login` to avoid confusion with the `name` attribute.
+
+Example name is `"Lukasz Gryglicki"`.
+
+Email pattern and name pattern are optional and `""` (empty) is assumed for them if not specified.
+
+Each pattern is a string and can be one of three possible types (and are checked tin this order):
+- `"name"` - exact match for repository name, GitHub login, email address, GitHub name.
+- `""` - (empty string) pattern is special and it matches missing property, property with null value or property with empty string value.
+- `"re:regexp"` - regular expression match for repository name, GitHub login, name, or email address.
+- `"*"` - matches all.
+
+So the format is like `"repository_pattern": "login_pattern;email_pattern;name_pattern"`. `;` is used as a separator.
+
+You can also specify multiple patterns so different set is used for multiple users - in such case configuration must start with `[`, end with `]` and be `||` separated.
+
+For example: `"[;*;copilot-swe-agent[bot];||re:(?i)^l(ukasz)?gryglicki$;*;re:Gryglicki]"`.
+
+Full format is like `"repository_pattern": "[login_pattern;email_pattern;name_pattern||..]"`.
+
+Other complex example: `"re:(?i)^repo\d*$": "[veerendra||re:(?i)^l(ukasz)?gryglicki$;lukaszgryglicki@o2.pl||*;*;Lukasz Gryglicki]"`.
+
+This matches one of:
+- GitHub login `veerendra` no matter the email and name.
+- GitHub login like lgryglicki, LukaszGryglicki and similar with email lukaszgryglicki@o2.pl, name doesn't matter.
+- GitHub name "Lukasz Gryglicki" email and login doesn't matter.
+
+There can be multiple entries under one Github Organization DynamoDB entry.
+
+Example:
+```
+{
+(...)
+ "organization_name": {
+ "S": "linuxfoundation"
+ },
+ "skip_cla": {
+ "M": {
+ "*": {
+ "S": ";re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]"
+ },
+ "re:(?i)^repo[0-9]+$": {
+ "S": "re:vee?rendra;*;*"
+ }
+ }
+ },
+(...)
+}
+```
+
+For example for `copilot-swe-agent[bot]` GitHub bot the exact values returned by GitHub are: id, login, name are all nulls, email is like this `198982749+Copilot@users.noreply.github.com`.
+
+Algorithm to match pattern is as follows:
+- First we check repository name for exact match. Repository name is without the organization name, so for `https://github.com/linuxfoundation/easycla` it is just `easycla`. If we find an entry in `skip_cla` for `easycla` that entry is used and we stop searching.
+- If no exact match is found, we check for regular expression match. Only keys starting with `re:` are considered. If we find a match, we use that entry and stop searching.
+- If no match is found, we check for `*` entry. If it exists, we use that entry and stop searching.
+- If no match is found, we don't skip CLA check for any author.
+- Now when we have the entry, it is in the following format: `login_pattern;email_pattern;name_pattern` or `"[login_pattern;email_pattern;name_pattern||...]" (array)`.
+- We check GitHub login, email address and name against the patterns. Algorithm is the same - login, email and name patterns can be either direct match ("" is a special case that also matches missing or null) or `re:regexp` or `*`.
+- If login, email and name match the patterns, we skip CLA check. If login, email or name is not set but the pattern is `*` it means hit.
+- So setting pattern to `login_pattern;*;*` means that we only check for login match and assume all emails and names are valid.
+- Any actor that matches any of the entries in the array will be skipped (logical OR).
+- If we set `repo_pattern` to `*` it means that this configuration applies to all repositories in the organization.
+- If there are also specific repository patterns, they will be used instead of `*` (fallback for all).
+
+
+There is a script that allows you to update the `skip_cla` property in the DynamoDB table. It is located in `utils/skip_cla_entry.sh`. You can run it like this:
+- `` MODE=mode ./utils/skip_cla_entry.sh 'org-name' 'repo-pattern' 'login-pattern;email-pattern;name_pattern' ``.
+- `` MODE=add-key ./utils/skip_cla_entry.sh 'sun-test-org' '*' ';*;copilot-swe-agent[bot]' ``.
+- Complex example: `` MODE=add-key ./utils/skip_cla_entry.sh 'sun-test-org' 're:(?i)^repo[0-9]+$' '[re:(?i)^l(ukasz)?gryglicki$;re:(?i)^l(ukasz)?gryglicki@;*||copilot-swe-agent[bot]]' ``.
+
+`MODE` can be one of:
+- `put-item`: Overwrites/adds the entire `skip_cla` property. Needs all 3 arguments org, repo, and pattern.
+- `add-key`: Adds or updates a key/value inside the `skip_cla` map (preserves other keys). Needs all 3 args.
+- `delete-key`: Removes a key from the `skip_cla` map. Needs 2 arguments: org and repo.
+- `delete-item`: Deletes the entire `skip_cla` from the item. Needs 1 argument: org.
+
+
+You can also use AWS CLI to update the `skip_cla` property. Here is an example command:
+
+To add a new `skip_cla` entry:
+
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item \
+ --table-name "cla-prod-github-orgs" \
+ --key '{"organization_name": {"S": "linuxfoundation"}}' \
+ --update-expression 'SET skip_cla = :val' \
+ --expression-attribute-values '{":val": {"M": {"re:^easycla":{"S":"some-github-login;*;*"}}}}'
+```
+
+To add a new key to an existing `skip_cla` entry (or replace the existing key):
+
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item \
+ --table-name "cla-prod-github-orgs" \
+ --key '{"organization_name": {"S": "linuxfoundation"}}' \
+ --update-expression "SET skip_cla.#repo = :val" \
+ --expression-attribute-names '{"#repo": "re:^easycla"}' \
+ --expression-attribute-values '{":val": {"S": "some-github-login;*;*"}}'
+```
+
+To delete a key from an existing `skip_cla` entry:
+
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item \
+ --table-name "cla-prod-github-orgs" \
+ --key '{"organization_name": {"S": "linuxfoundation"}}' \
+ --update-expression "REMOVE skip_cla.#repo" \
+ --expression-attribute-names '{"#repo": "re:^easycla"}'
+```
+
+To delete the entire `skip_cla` entry:
+
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item \
+ --table-name "cla-prod-github-orgs" \
+ --key '{"organization_name": {"S": "linuxfoundation"}}' \
+ --update-expression "REMOVE skip_cla"
+```
+
+To see given organization's entry: `./utils/scan.sh github-orgs organization_name sun-test-org`.
+
+Or using AWS CLI:
+
+```
+aws --profile "lfproduct-prod" dynamodb scan --table-name "cla-prod-github-orgs" --filter-expression "contains(organization_name,:v)" --expression-attribute-values "{\":v\":{\"S\":\"linuxfoundation\"}}" --max-items 100 | jq -r '.Items'
+```
+
+To check for log entries related to skipping CLA check, you can use the following command: `` STAGE=dev DTFROM='1 hour ago' DTTO='1 second ago' ./utils/search_aws_log_group.sh 'cla-backend-dev-githubactivity' 'skip_cla' ``.
+
+# Example setup on prod
+
+To add first `skip_cla` value for an organization:
+```
+aws --profile lfproduct-prod --region us-east-1 dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "open-telemetry"}}' --update-expression 'SET skip_cla = :val' --expression-attribute-values '{":val": {"M": {"otel-arrow":{"S":";re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]"}}}}'
+aws --profile lfproduct-prod --region us-east-1 dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "openfga"}}' --update-expression 'SET skip_cla = :val' --expression-attribute-values '{":val": {"M": {"vscode-ext":{"S":";re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]"}}}}'
+aws --profile lfproduct-prod --region us-east-1 dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "eslint"}}' --update-expression 'SET skip_cla = :val' --expression-attribute-values '{":val": {"M": {"*":{"S":"[Copilot;re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]||;re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]||Copilot;re:^\\d+\\+copilot@users\\.noreply\\.github\\.com$;Copilot]"}}}}'
+```
+
+To add additional repositories entries without overwriting the existing `skip_cla` value:
+```
+aws --profile lfproduct-prod --region us-east-1 dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "open-telemetry"}}' --update-expression 'SET skip_cla.#repo = :val' --expression-attribute-names '{"#repo": "*"}' --expression-attribute-values '{":val": {"S": ";re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]"}}'
+aws --profile lfproduct-prod --region us-east-1 dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "openfga"}}' --update-expression 'SET skip_cla.#repo = :val' --expression-attribute-names '{"#repo": "*"}' --expression-attribute-values '{":val": {"S": ";re:^\\d+\\+Copilot@users\\.noreply\\.github\\.com$;copilot-swe-agent[bot]"}}'
+```
+
+To delete a specific repo entry from `skip_cla`:
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "open-telemetry"}}' --update-expression 'REMOVE skip_cla.#repo' --expression-attribute-names '{"#repo": "*"}'
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "openfga"}}' --update-expression 'REMOVE skip_cla.#repo' --expression-attribute-names '{"#repo": "*"}'
+```
+
+To delete the entire `skip_cla` attribute:
+```
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "open-telemetry"}}' --update-expression 'REMOVE skip_cla'
+aws --profile "lfproduct-prod" --region "us-east-1" dynamodb update-item --table-name "cla-prod-github-orgs" --key '{"organization_name": {"S": "openfga"}}' --update-expression 'REMOVE skip_cla'
+```
+
+To check values:
+```
+aws --profile "lfproduct-prod" dynamodb scan --table-name "cla-prod-github-orgs" --filter-expression "contains(organization_name,:v)" --expression-attribute-values "{\":v\":{\"S\":\"open-telemetry\"}}" --max-items 100 | jq -r '.Items'
+aws --profile "lfproduct-prod" dynamodb scan --table-name "cla-prod-github-orgs" --filter-expression "contains(organization_name,:v)" --expression-attribute-values "{\":v\":{\"S\":\"openfga\"}}" --max-items 100 | jq -r '.Items'
+aws --profile "lfproduct-prod" dynamodb scan --table-name "cla-prod-github-orgs" --filter-expression "contains(organization_name,:v)" --expression-attribute-values "{\":v\":{\"S\":\"open-telemetry\"}}" --max-items 100 | jq -r '.Items[0].skip_cla.M["otel-arrow"]["S"]'
+aws --profile "lfproduct-prod" dynamodb scan --table-name "cla-prod-github-orgs" --filter-expression "contains(organization_name,:v)" --expression-attribute-values "{\":v\":{\"S\":\"openfga\"}}" --max-items 100 | jq -r '.Items[0].skip_cla.M["vscode-ext"]["S"]'
+```
+
+Typical adding a new entry for an organization:
+```
+STAGE=prod MODE=add-key DEBUG=1 ./utils/skip_cla_entry.sh 'open-telemetry' 'opentelemetry-rust' ';re:^\d+\+Copilot@users\.noreply\.github\.com$;copilot-swe-agent[bot]'
+```
+
diff --git a/CHECK_PR.md b/CHECK_PR.md
new file mode 100644
index 000000000..f1070d341
--- /dev/null
+++ b/CHECK_PR.md
@@ -0,0 +1,8 @@
+# How to check why EasyCLA is missing signed agreement
+
+1) Open GitHub PR and check for user name: `` select * from FIVETRAN_INGEST.DYNAMODB_PRODUCT_US_EAST_1.CLA_PROD_USERS where data:user_github_username = ' %s included the following message in the request: %s
Hello %s,
This is a notification email from EasyCLA regarding the project %s.
-%s (%s) has requested to be added to the Approved List as an authorized contributor from -%s to the project %s. You are receiving this message as a CLA Manager from %s for -%s.
-%s -If you want to add them to the Approved List, please -log into the EasyCLA Corporate -Console, where you can approve this user's request by selecting the 'Manage Approved List' and adding the -contributor's email, the contributor's entire email domain, their GitHub ID or the entire GitHub Organization for the -repository. This will permit them to begin contributing to %s on behalf of %s.
-If you are not certain whether to add them to the Approved List, please reach out to them directly to discuss.
-%s -%s`, - recipientName, projectName, contributorName, contributorEmail, - companyName, projectName, companyName, projectName, - optionalMessage, s.corpConsoleURL, - companyModel.CompanyID, projectName, companyName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + subject := fmt.Sprintf("EasyCLA: Request to Authorize %s for %s", emailParams.ContributorName, projectName) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderRequestToAuthorizeTemplate(s.emailTemplateService, claGroupModel.Version, claGroupModel.ProjectExternalID, emailParams) + if err != nil { + log.Warnf("rendering email template : %s failed : %v", emails.RequestToAuthorizeTemplateName, err) + return + } + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -295,20 +383,17 @@ repository. This will permit them to begin contributing to %s on behalf of %s. } // sendRequestRejectedEmailToRecipient generates and sends an email to the specified recipient -func (s service) sendRequestRejectedEmailToRecipient(companyModel *models.Company, claGroupModel *models.ClaGroup, signature *models.Signature, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName +func (s service) sendRequestRejectedEmailToRecipient(emailParams emails.CommonEmailParams, claGroupModel *models.ClaGroup, signature *models.Signature) { projectName := claGroupModel.ProjectName - var claManagerText = "" - claManagerText += "Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-Your request to get added to the approval list from %s for %s was denied by one of the existing CLA Managers. -If you have further questions about this denial, please contact one of the existing CLA Managers from -%s for %s:
-%s -%s -%s`, - recipientName, projectName, - companyName, projectName, companyName, projectName, - claManagerText, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderApprovalListRejectedTemplate( + s.emailTemplateService, claGroupModel.Version, claGroupModel.ProjectExternalID, emails.ApprovalListRejectedTemplateParams{ + CommonEmailParams: emailParams, + CLAManagers: emailCLAManagerParams, + }) + if err != nil { + log.Warnf("rendering email failed for : %s : %v", emails.ApprovalListRejectedTemplateName, err) + return + } + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -348,40 +430,42 @@ If you have further questions about this denial, please contact one of the exist } } -func requestApprovedEmailToRecipientContent(companyModel *models.Company, claGroupModel *models.ClaGroup, recipientName, recipientAddress string) (string, string, []string) { - companyName := companyModel.CompanyName +func (s service) sendRequestApprovedEmailToRecipient(ctx context.Context, emailParams emails.CommonEmailParams, claUser user.CLAUser, projectSFIDs []string) { + f := logrus.Fields{ + "functionName": "v1.approval_list.service.sendRequestApprovedEmailToRecipient", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyName": emailParams.CompanyName, + "recipientName": emailParams.RecipientName, + "recipientAddress": emailParams.RecipientAddress, + } + companyName := emailParams.CompanyName // subject string, body string, recipients []string subject := fmt.Sprintf("EasyCLA: Approved List Request Accepted for %s", companyName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the company %s.
-You have now been added to the approval list for %s.
-To get started, please navigate back to GitHub or Gerrit and start the authorization process. Once you select the -authorization link, you will be directed to the EasyCLA Contributor Console. GitHub users will need to authorize the -tool to see your GitHub user name and email. Gerrit users will first need to log in with their LF Account. On the -console landing page, select the corporate agreement option. To finish, search and select your company to acknowledge -your association with your company. This will complete the authorization process. For GitHub users, your pull request -will refresh and confirm that you are authorized. For Gerrit users, please log out of the UI and back in to complete the -authorization.
-%s -%s`, - recipientName, companyName, - companyName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), - utils.GetEmailSignOffContent()) - - return subject, body, recipients -} + recipients := []string{emailParams.RecipientAddress} + + approver := "" + if claUser.LFUsername != "" { + approver = claUser.LFUsername + } else if claUser.LFEmail != "" { + approver = claUser.LFEmail + } else if claUser.Emails != nil { + approver = claUser.Emails[0] + } -// sendRequestApprovedEmailToRecipient generates and sends an email to the specified recipient -func sendRequestApprovedEmailToRecipient(companyModel *models.Company, claGroupModel *models.ClaGroup, recipientName, recipientAddress string) { - subject, body, recipients := requestApprovedEmailToRecipientContent(companyModel, claGroupModel, recipientName, recipientAddress) - err := utils.SendEmail(subject, body, recipients) + body, err := emails.RenderApprovalListTemplate( + s.emailTemplateService, projectSFIDs, emails.ApprovalListApprovedTemplateParams{ + CommonEmailParams: emailParams, + Approver: approver, + }) if err != nil { - log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) + log.WithFields(f).Warnf("rendering email failed for : %s : %v", emails.ApprovalListApprovedTemplateName, err) + return + } + err = utils.SendEmail(subject, body, recipients) + if err != nil { + log.WithFields(f).Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { - log.Debugf("sent email with subject: %s to recipients: %+v", subject, recipients) + log.WithFields(f).Debugf("sent email with subject: %s to recipients: %+v", subject, recipients) } } diff --git a/cla-backend-go/approval_list/service_test.go b/cla-backend-go/approval_list/service_test.go deleted file mode 100644 index ebc053a70..000000000 --- a/cla-backend-go/approval_list/service_test.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright The Linux Foundation and each contributor to CommunityBridge. -// SPDX-License-Identifier: MIT - -package approval_list - -import ( - "testing" - - "github.com/communitybridge/easycla/cla-backend-go/gen/models" - "github.com/stretchr/testify/assert" -) - -func TestRequestApprovedEmailToRecipientContent(t *testing.T) { - subject, body, recipients := requestApprovedEmailToRecipientContent( - &models.Company{ - CompanyName: "gardenerLtd"}, - &models.ClaGroup{Version: "v2"}, - "john", - "john@john.com") - - assert.Equal(t, "EasyCLA: Approved List Request Accepted for gardenerLtd", subject) - assert.Equal(t, []string{"john@john.com"}, recipients) - assert.Contains(t, body, "Hello john,") - assert.Contains(t, body, "This is a notification email from EasyCLA regarding the company gardenerLtd") - assert.Contains(t, body, "You have now been added to the approval list for gardenerLtd") -} diff --git a/cla-backend-go/auth/auth0.go b/cla-backend-go/auth/auth0.go index 71d966aa2..dcaa3ef98 100644 --- a/cla-backend-go/auth/auth0.go +++ b/cla-backend-go/auth/auth0.go @@ -9,7 +9,8 @@ import ( "net/http" "path" - "github.com/dgrijalva/jwt-go" + "github.com/golang-jwt/jwt/v4" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" ) // Validator data model @@ -23,7 +24,7 @@ type Validator struct { } // NewAuthValidator creates a new auth0 validator based on the specified parameters -func NewAuthValidator(domain, clientID, usernameClaim, algorithm string) (Validator, error) { // nolint +func NewAuthValidator(domain, clientID, usernameClaim, nameClaim, emailClaim, algorithm string) (Validator, error) { // nolint if domain == "" { return Validator{}, errors.New("missing Domain") } @@ -42,8 +43,8 @@ func NewAuthValidator(domain, clientID, usernameClaim, algorithm string) (Valida usernameClaim: usernameClaim, algorithm: algorithm, wellKnownURL: "https://" + path.Join(domain, ".well-known/jwks.json"), - nameClaim: "name", - emailClaim: "email", + nameClaim: nameClaim, + emailClaim: emailClaim, } return validator, nil @@ -93,7 +94,12 @@ func (av Validator) getPemCert(token *jwt.Token) (interface{}, error) { if err != nil { return "", err } - defer resp.Body.Close() + defer func() { + closeErr := resp.Body.Close() + if closeErr != nil { + log.WithError(closeErr).Warn("problem closing response body") + } + }() var j = jwks{} err = json.NewDecoder(resp.Body).Decode(&j) diff --git a/cla-backend-go/auth/authorizer.go b/cla-backend-go/auth/authorizer.go index c973ed894..cac3d83e7 100644 --- a/cla-backend-go/auth/authorizer.go +++ b/cla-backend-go/auth/authorizer.go @@ -5,15 +5,16 @@ package auth import ( "errors" + "fmt" "strings" "github.com/sirupsen/logrus" swagerrors "github.com/go-openapi/errors" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/user" + "github.com/linuxfoundation/easycla/cla-backend-go/user" ) const ( @@ -50,7 +51,7 @@ func NewAuthorizer(authValidator Validator, userPermissioner UserPermissioner) A // SecurityAuth creates a new CLA user based on the token and scopes func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, error) { f := logrus.Fields{ - "functionName": "auth.SecurityAuth", + "functionName": "auth.authorizer.SecurityAuth", "scopes": strings.Join(scopes, ","), } // This handler is called by the runtime whenever a route needs authentication @@ -59,17 +60,22 @@ func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, // the list of scopes mentioned by the spec for this route. // Verify the token is valid + // LG:to skip verification log.WithFields(f).Debug("verifying token...") claims, err := a.authValidator.VerifyToken(token) if err != nil { log.WithFields(f).WithError(err).Warnf("SecurityAuth - verify token error: %+v", err) if strings.Contains(strings.ToLower(err.Error()), "expired") { - return nil, swagerrors.New(401, err.Error()) + return nil, swagerrors.New(401, "%s", err.Error()) } return nil, err } + f["claims"] = fmt.Sprintf("%+v", claims) // Get the username from the token claims + // LG: for V3 endpoints comment this out and set: username, name and email manually for local testing. + // username, name, email := "user", "Name Surname", "example@gmail.com" + // username, name, email := "mock-user-go-20250522", "Mock User Go 2025-05-22", "u20250522@mock.user.go.pl" usernameClaim, ok := claims[a.authValidator.usernameClaim] if !ok { log.WithFields(f).Warnf("username not found in claims with key: %s", a.authValidator.usernameClaim) @@ -83,6 +89,10 @@ func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, } f["username"] = username + // LG: to allow local testing + // a.authValidator.nameClaim = "http://lfx.dev/claims/username" + // a.authValidator.emailClaim = "http://lfx.dev/claims/email" + nameClaim, ok := claims[a.authValidator.nameClaim] if !ok { log.WithFields(f).Warnf("name not found: %+v", a.authValidator.nameClaim) @@ -108,6 +118,7 @@ func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, return nil, errors.New("invalid email") } f["email"] = email + // LG:end // Get User by LFID log.WithFields(f).Debugf("loading user and profiles by LFID: %s", username) @@ -123,6 +134,7 @@ func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, } return nil, err } + //log.WithFields(f).Debugf("user loaded : %+v with scopes : %+v", lfuser, scopes) for _, scope := range scopes { switch Scope(scope) { @@ -149,5 +161,6 @@ func (a Authorizer) SecurityAuth(token string, scopes []string) (*user.CLAUser, } } + //log.WithFields(f).Debugf("returning user from auth : %+v", lfuser) return &lfuser, nil } diff --git a/cla-backend-go/cla_manager/handlers.go b/cla-backend-go/cla_manager/handlers.go index df311de9d..4cf2b11c8 100644 --- a/cla-backend-go/cla_manager/handlers.go +++ b/cla-backend-go/cla_manager/handlers.go @@ -7,23 +7,33 @@ import ( "context" "fmt" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations/cla_manager" + service2 "github.com/linuxfoundation/easycla/cla-backend-go/project/service" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/go-openapi/strfmt" + + "github.com/LF-Engineering/lfx-kit/auth" + + user_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/user-service" + "github.com/sirupsen/logrus" + + "github.com/linuxfoundation/easycla/cla-backend-go/emails" + + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/cla_manager" + + "github.com/linuxfoundation/easycla/cla-backend-go/utils" "github.com/aws/aws-sdk-go/aws" - sigAPI "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations/signatures" - "github.com/communitybridge/easycla/cla-backend-go/signatures" - - "github.com/communitybridge/easycla/cla-backend-go/company" - "github.com/communitybridge/easycla/cla-backend-go/events" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/project" - "github.com/communitybridge/easycla/cla-backend-go/user" - "github.com/communitybridge/easycla/cla-backend-go/users" + sigAPI "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/go-openapi/runtime/middleware" + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/user" + "github.com/linuxfoundation/easycla/cla-backend-go/users" ) // isValidUser is a helper function to determine if the user object is valid @@ -32,7 +42,7 @@ func isValidUser(claUser *user.CLAUser) bool { } // Configure is the API handler routine for the CLA manager routes -func Configure(api *operations.ClaAPI, service IService, companyService company.IService, projectService project.Service, usersService users.Service, sigService signatures.SignatureService, eventsService events.Service, corporateConsoleURL string) { // nolint +func Configure(api *operations.ClaAPI, service IService, companyService company.IService, projectService service2.Service, usersService users.Service, sigService signatures.SignatureService, eventsService events.Service, emailSvc emails.EmailTemplateService) { // nolint api.ClaManagerCreateCLAManagerRequestHandler = cla_manager.CreateCLAManagerRequestHandlerFunc(func(params cla_manager.CreateCLAManagerRequestParams, claUser *user.CLAUser) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint @@ -160,16 +170,16 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Send an event - eventsService.LogEvent(&events.LogEventArgs{ - EventType: events.ClaManagerAccessRequestCreated, - ProjectID: params.ProjectID, - ClaGroupModel: claGroupModel, - CompanyID: params.CompanyID, - CompanyModel: companyModel, - LfUsername: params.Body.UserLFID, - UserID: params.Body.UserLFID, - UserModel: userModel, - ExternalProjectID: claGroupModel.ProjectExternalID, + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ + EventType: events.ClaManagerAccessRequestCreated, + ProjectID: params.ProjectID, + ClaGroupModel: claGroupModel, + CompanyID: params.CompanyID, + CompanyModel: companyModel, + LfUsername: params.Body.UserLFID, + UserID: params.Body.UserLFID, + UserModel: userModel, + ProjectSFID: claGroupModel.ProjectExternalID, EventData: &events.CLAManagerRequestCreatedEventData{ RequestID: request.RequestID, CompanyName: companyModel.CompanyName, @@ -182,9 +192,15 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. // Send email to each manager for _, manager := range claManagers { - sendRequestAccessEmailToCLAManagers(companyModel, claGroupModel, - params.Body.UserName, params.Body.UserEmail, - manager.Username, manager.LfEmail) + sendRequestAccessEmailToCLAManagers(emailSvc, emails.RequestAccessToCLAManagersTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: manager.Username, + RecipientAddress: manager.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, + RequesterName: params.Body.UserName, + RequesterEmail: params.Body.UserEmail, + }, claGroupModel) } return cla_manager.NewCreateCLAManagerRequestOK().WithXRequestID(reqID).WithPayload(request) @@ -252,10 +268,18 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "cla_manager.handler.ClaManagerApproveCLAManagerRequestHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + "CompanyID": params.CompanyID, + "RequestID": params.RequestID, + } + companyModel, companyErr := companyService.GetCompany(ctx, params.CompanyID) if companyErr != nil || companyModel == nil { msg := buildErrorMessageForApprove(params, companyErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewCreateCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -265,7 +289,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. claGroupModel, projectErr := projectService.GetCLAGroupByID(ctx, params.ProjectID) if projectErr != nil || claGroupModel == nil { msg := buildErrorMessageForApprove(params, projectErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewCreateCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -282,14 +306,14 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. }) if sigErr != nil || sigModels == nil { msg := buildErrorMessageForApprove(params, sigErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewApproveCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: "CLA Manager Approve Request - error reading CCLA Signatures - " + msg, Code: "400", }) } if len(sigModels.Signatures) > 1 { - log.Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", + log.WithFields(f).Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", params.CompanyID, params.ProjectID, params.RequestID) } @@ -307,7 +331,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. request, err := service.ApproveRequest(params.CompanyID, params.ProjectID, params.RequestID) if err != nil { msg := buildErrorMessageForApprove(params, err) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewApproveCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -315,10 +339,10 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Update the signature ACL - _, aclErr := sigService.AddCLAManager(ctx, sigModel.SignatureID.String(), request.UserID) + _, aclErr := sigService.AddCLAManager(ctx, sigModel.SignatureID, request.UserID) if aclErr != nil { msg := buildErrorMessageForApprove(params, aclErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewApproveCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -326,7 +350,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Send an event - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.ClaManagerAccessRequestApproved, ProjectID: params.ProjectID, CompanyID: params.CompanyID, @@ -344,12 +368,25 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. // Notify CLA Managers - send email to each manager for _, manager := range claManagers { - sendRequestApprovedEmailToCLAManagers(companyModel, claGroupModel, request.UserName, request.UserEmail, - manager.Username, manager.LfEmail) + sendRequestApprovedEmailToCLAManagers(emailSvc, emails.RequestApprovedToCLAManagersTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: manager.Username, + RecipientAddress: manager.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, + RequesterName: request.UserName, + RequesterEmail: request.UserEmail, + }, claGroupModel) } // Notify the requester - sendRequestApprovedEmailToRequester(companyModel, claGroupModel, request.UserName, request.UserEmail) + sendRequestApprovedEmailToRequester(emailSvc, emails.RequestApprovedToRequesterTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: request.UserName, + RecipientAddress: request.UserEmail, + CompanyName: companyModel.CompanyName, + }, + }, claGroupModel) return cla_manager.NewCreateCLAManagerRequestOK().WithXRequestID(reqID).WithPayload(request) }) @@ -358,11 +395,18 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. api.ClaManagerDenyCLAManagerRequestHandler = cla_manager.DenyCLAManagerRequestHandlerFunc(func(params cla_manager.DenyCLAManagerRequestParams, claUser *user.CLAUser) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "cla_manager.handler.ClaManagerDenyCLAManagerRequestHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + "CompanyID": params.CompanyID, + "RequestID": params.RequestID, + } companyModel, companyErr := companyService.GetCompany(ctx, params.CompanyID) if companyErr != nil || companyModel == nil { msg := buildErrorMessageForDeny(params, companyErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDenyCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -372,7 +416,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. claGroupModel, projectErr := projectService.GetCLAGroupByID(ctx, params.ProjectID) if projectErr != nil || claGroupModel == nil { msg := buildErrorMessageForDeny(params, projectErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDenyCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -389,14 +433,14 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. }) if sigErr != nil || sigModels == nil { msg := buildErrorMessageForDeny(params, sigErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewApproveCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: "CLA Manager Deny Request - error reading CCLA Signatures - " + msg, Code: "400", }) } if len(sigModels.Signatures) > 1 { - log.Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", + log.WithFields(f).Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", params.CompanyID, params.ProjectID, params.RequestID) } @@ -413,7 +457,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. request, err := service.DenyRequest(params.CompanyID, params.ProjectID, params.RequestID) if err != nil { msg := buildErrorMessageForDeny(params, err) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDenyCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -421,7 +465,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Send an event - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.ClaManagerAccessRequestDenied, ProjectID: params.ProjectID, CompanyID: params.CompanyID, @@ -439,12 +483,23 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. // Notify CLA Managers - send email to each manager for _, manager := range claManagers { - sendRequestDeniedEmailToCLAManagers(companyModel, claGroupModel, request.UserName, request.UserEmail, - manager.Username, manager.LfEmail) + sendRequestDeniedEmailToCLAManagers(emailSvc, emails.RequestDeniedToCLAManagersTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: manager.Username, + RecipientAddress: manager.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, + RequesterName: request.UserName, + RequesterEmail: request.UserEmail, + }, claGroupModel) } // Notify the requester - sendRequestDeniedEmailToRequester(companyModel, claGroupModel, request.UserName, request.UserEmail) + sendRequestDeniedEmailToRequester(emailSvc, emails.CommonEmailParams{ + RecipientName: request.UserName, + RecipientAddress: request.UserEmail, + CompanyName: companyModel.CompanyName, + }, claGroupModel) return cla_manager.NewCreateCLAManagerRequestOK().WithPayload(request) }) @@ -453,6 +508,13 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. api.ClaManagerDeleteCLAManagerRequestHandler = cla_manager.DeleteCLAManagerRequestHandlerFunc(func(params cla_manager.DeleteCLAManagerRequestParams, claUser *user.CLAUser) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "cla_manager.handler.ClaManagerDeleteCLAManagerRequestHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + "CompanyID": params.CompanyID, + "RequestID": params.RequestID, + } // Make sure the company id exists... companyModel, companyErr := companyService.GetCompany(ctx, params.CompanyID) @@ -468,7 +530,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. claGroupModel, projectErr := projectService.GetCLAGroupByID(ctx, params.ProjectID) if projectErr != nil || claGroupModel == nil { msg := buildErrorMessageForDelete(params, projectErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDenyCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -479,7 +541,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. request, err := service.GetRequest(params.RequestID) if err != nil { msg := buildErrorMessageForDelete(params, err) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDeleteCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -488,7 +550,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. if request == nil { msg := buildErrorMessageForDelete(params, err) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDeleteCLAManagerRequestNotFound().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "404", @@ -505,14 +567,16 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. }) if sigErr != nil || sigModels == nil { msg := buildErrorMessageForDelete(params, sigErr) - log.Warn(msg) - return cla_manager.NewDeleteCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - CLA Manager Delete Request - error reading CCLA Signatures - " + msg, - Code: "400", - }) + log.WithFields(f).Warn(msg) + return cla_manager.NewDeleteCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse( + utils.ErrorResponseBadRequest( + reqID, + "CLA Manager Delete Request - error reading CCLA Signatures - "+msg, + ))) } if len(sigModels.Signatures) > 1 { - log.Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", + log.WithFields(f).Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s for request ID: %s", params.CompanyID, params.ProjectID, params.RequestID) } @@ -521,7 +585,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. if !currentUserInACL(claUser, claManagers) { msg := fmt.Sprintf("EasyCLA - 401 Unauthorized - CLA Manager %s / %s / %s not authorized to delete requests for company ID: %s, project ID: %s", claUser.UserID, claUser.Name, claUser.LFEmail, params.CompanyID, params.ProjectID) - log.Debug(msg) + log.WithFields(f).Debug(msg) return cla_manager.NewDeleteCLAManagerRequestUnauthorized().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "401", @@ -532,7 +596,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. deleteErr := service.DeleteRequest(params.RequestID) if deleteErr != nil { msg := buildErrorMessageForDelete(params, deleteErr) - log.Warn(msg) + log.WithFields(f).Warn(msg) return cla_manager.NewDeleteCLAManagerRequestBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Message: msg, Code: "400", @@ -540,7 +604,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Send an event - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.ClaManagerAccessRequestDeleted, ProjectID: params.ProjectID, CompanyID: params.CompanyID, @@ -556,41 +620,72 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. }, }) - log.Debug("CLA Manager Delete - Returning Success") + log.WithFields(f).Debug("CLA Manager Delete - Returning Success") return cla_manager.NewDeleteCLAManagerRequestNoContent().WithXRequestID(reqID) }) api.ClaManagerAddCLAManagerHandler = cla_manager.AddCLAManagerHandlerFunc(func(params cla_manager.AddCLAManagerParams, claUser *user.CLAUser) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "cla_manager.handler.ClaManagerAddCLAManagerHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + "CompanyID": params.CompanyID, + "UserLFID": params.Body.UserLFID, + "UserEmail": params.Body.UserEmail, + "UserName": params.Body.UserName, + } + log.WithFields(f).Debug("looking up user by user id...") userModel, userErr := usersService.GetUserByLFUserName(params.Body.UserLFID) if userErr != nil || userModel == nil { - msg := fmt.Sprintf("User lookup for user by LFID: %s failed ", params.Body.UserLFID) - log.Warn(msg) - return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - Add CLA Manager - error getting user - " + msg, - Code: "400", - }) + log.WithFields(f).Warnf("Add CLA Manager - user lookup by LFID: %s failed - attempting to lookup in SF...", params.Body.UserLFID) + userServiceClient := user_service.GetClient() + sfdcUserObject, userServiceLookupErr := userServiceClient.GetUserByUsername(params.Body.UserLFID) + if userServiceLookupErr != nil || sfdcUserObject == nil { + msg := fmt.Sprintf("Add CLA Manager - user lookup by LFID: %s failed ", params.Body.UserLFID) + log.WithFields(f).Warn(msg) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequestWithError(reqID, msg, userServiceLookupErr))) + } + + _, nowStr := utils.CurrentTime() + userModel, userErr = usersService.CreateUser(&models.User{ + Admin: false, + DateCreated: nowStr, + DateModified: nowStr, + Emails: userServiceClient.EmailsToSlice(sfdcUserObject), + GithubUsername: sfdcUserObject.GithubID, //this is the github username + LfEmail: strfmt.Email(userServiceClient.GetPrimaryEmail(sfdcUserObject)), + LfUsername: sfdcUserObject.Username, + Note: "created from SF record", + UserExternalID: sfdcUserObject.ID, + Username: sfdcUserObject.Username, + Version: "v1", + }, claUser) + if userErr != nil || userModel == nil { + msg := fmt.Sprintf("Add CLA Manager - user lookup by LFID: %s failed ", params.Body.UserLFID) + log.WithFields(f).Warn(msg) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequestWithError(reqID, msg, userErr))) + } } + companyModel, companyErr := companyService.GetCompany(ctx, params.CompanyID) if companyErr != nil || companyModel == nil { - msg := fmt.Sprintf("User lookup for company by ID: %s failed ", params.CompanyID) + msg := fmt.Sprintf("Add CLA Manager - error getting company by ID: %s failed ", params.CompanyID) log.Warn(msg) - return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - Add CLA Manager - error getting company - " + msg, - Code: "400", - }) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequest(reqID, msg))) } claGroupModel, projectErr := projectService.GetCLAGroupByID(ctx, params.ProjectID) if projectErr != nil || claGroupModel == nil { - msg := fmt.Sprintf("User lookup for project by ID: %s failed ", params.ProjectID) + msg := fmt.Sprintf("Add CLA Manager - error getting project - lookup for project by ID: %s failed ", params.ProjectID) log.Warn(msg) - return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - Add CLA Manager - error getting project - " + msg, - Code: "400", - }) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequest(reqID, msg))) } // Look up signature ACL to ensure the user is allowed to add CLA managers @@ -604,10 +699,8 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. if sigErr != nil || sigModels == nil { msg := buildErrorMessageAddManager("Add CLA Manager - signature lookup error", params, sigErr) log.Warn(msg) - return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - Add CLA Manager - error reading CCLA Signatures - " + msg, - Code: "400", - }) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequest(reqID, msg))) } if len(sigModels.Signatures) > 1 { log.Warnf("returned multiple CCLA signature models for company ID: %s, project ID: %s", @@ -620,21 +713,17 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. msg := fmt.Sprintf("EasyCLA - 401 Unauthorized - User %s / %s / %s is not authorized to add a CLA Manager for company ID: %s, project ID: %s", claUser.UserID, claUser.Name, claUser.LFEmail, params.CompanyID, params.ProjectID) log.Debug(msg) - return cla_manager.NewAddCLAManagerUnauthorized().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: msg, - Code: "401", - }) + return cla_manager.NewAddCLAManagerUnauthorized().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseUnauthorized(reqID, msg))) } // Audit Event sent from service upon success - signature, addErr := service.AddClaManager(ctx, params.CompanyID, params.ProjectID, params.Body.UserLFID) + signature, addErr := service.AddClaManager(ctx, ToAuthUser(claUser), params.CompanyID, params.ProjectID, params.Body.UserLFID, "") if addErr != nil { msg := buildErrorMessageAddManager("Add CLA Manager - Service Error", params, addErr) log.Warn(msg) - return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: msg, - Code: "400", - }) + return cla_manager.NewAddCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequest(reqID, msg))) } return cla_manager.NewAddCLAManagerOK().WithXRequestID(reqID).WithPayload(signature) @@ -644,16 +733,50 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. api.ClaManagerDeleteCLAManagerHandler = cla_manager.DeleteCLAManagerHandlerFunc(func(params cla_manager.DeleteCLAManagerParams, claUser *user.CLAUser) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "cla_manager.handler.ClaManagerDeleteCLAManagerHandler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "projectID": params.ProjectID, + "CompanyID": params.CompanyID, + "UserLFID": params.UserLFID, + } + + log.WithFields(f).Debug("looking up user by user id...") userModel, userErr := usersService.GetUserByLFUserName(params.UserLFID) if userErr != nil || userModel == nil { - msg := fmt.Sprintf("User lookup for user by LFID: %s failed ", params.UserLFID) - log.Warn(msg) - return cla_manager.NewDeleteCLAManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ - Message: "EasyCLA - 400 Bad Request - Delete CLA Manager - error getting user - " + msg, - Code: "400", - }) + log.WithFields(f).Warnf("user lookup by LFID: %s failed - attempting to lookup in SF...", params.UserLFID) + userServiceClient := user_service.GetClient() + sfdcUserObject, userServiceLookupErr := userServiceClient.GetUserByUsername(params.UserLFID) + if userServiceLookupErr != nil || sfdcUserObject == nil { + msg := fmt.Sprintf("Delete CLA Manager - user lookup by LFID: %s failed ", params.UserLFID) + log.WithFields(f).Warn(msg) + return cla_manager.NewDeleteCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequest(reqID, msg))) + } + + _, nowStr := utils.CurrentTime() + userModel, userErr = usersService.CreateUser(&models.User{ + Admin: false, + DateCreated: nowStr, + DateModified: nowStr, + Emails: userServiceClient.EmailsToSlice(sfdcUserObject), + GithubUsername: sfdcUserObject.GithubID, //this is the github username + LfEmail: strfmt.Email(userServiceClient.GetPrimaryEmail(sfdcUserObject)), + LfUsername: sfdcUserObject.Username, + Note: "created from SF record", + UserExternalID: sfdcUserObject.ID, + Username: sfdcUserObject.Username, + Version: "v1", + }, claUser) + if userErr != nil || userModel == nil { + msg := fmt.Sprintf("Add CLA Manager - user lookup by LFID: %s failed ", params.UserLFID) + log.WithFields(f).Warn(msg) + return cla_manager.NewDeleteCLAManagerBadRequest().WithXRequestID(reqID).WithPayload( + utils.ToV1ErrorResponse(utils.ErrorResponseBadRequestWithError(reqID, msg, userErr))) + } } + companyModel, companyErr := companyService.GetCompany(ctx, params.CompanyID) if companyErr != nil || companyModel == nil { msg := fmt.Sprintf("User lookup for company by ID: %s failed ", params.CompanyID) @@ -707,7 +830,7 @@ func Configure(api *operations.ClaAPI, service IService, companyService company. } // Audit Event sent from service upon success - signature, deleteErr := service.RemoveClaManager(ctx, params.CompanyID, params.ProjectID, params.UserLFID) + signature, deleteErr := service.RemoveClaManager(ctx, ToAuthUser(claUser), params.CompanyID, params.ProjectID, params.UserLFID, "") if deleteErr != nil { msg := buildErrorMessageDeleteManager("EasyCLA - 400 Bad Request - Delete CLA Manager - Service Error", params, deleteErr) @@ -794,34 +917,21 @@ func buildErrorMessageDeleteManager(errPrefix string, params cla_manager.DeleteC } // sendRequestAccessEmailToCLAManagers sends the request access email to the specified CLA Managers -func sendRequestAccessEmailToCLAManagers(companyModel *models.Company, claGroupModel *models.ClaGroup, requesterName, requesterEmail, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName +func sendRequestAccessEmailToCLAManagers(emailSvc emails.EmailTemplateService, emailParams emails.RequestAccessToCLAManagersTemplateParams, claGroupModel *models.ClaGroup) { + companyName := emailParams.CompanyName projectName := claGroupModel.ProjectName // subject string, body string, recipients []string subject := fmt.Sprintf("EasyCLA: New CLA Manager Access Request for %s on %s", companyName, projectName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-You are currently listed as a CLA Manager from %s for the project %s. This means that you are able to maintain the -list of employees allowed to contribute to %s on behalf of your company, as well as view and manage the list of -your company’s CLA Managers for %s.
-%s (%s) has requested to be added as another CLA Manager from %s for %s. This would permit them to maintain the -lists of approved contributors and CLA Managers as well.
-If you want to permit this, please log into the EasyCLA Corporate Console, -select your company, then select the %s project. From the CLA Manager requests, you can approve this user as an -additional CLA Manager.
-%s -%s -`, - recipientName, projectName, - companyName, projectName, projectName, projectName, - requesterName, requesterEmail, companyName, projectName, - utils.GetCorporateURL(claGroupModel.Version == utils.V2), projectName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderRequestAccessToCLAManagersTemplate( + emailSvc, claGroupModel.Version, claGroupModel.ProjectExternalID, emailParams) + if err != nil { + log.Warnf("rendering email template : %s failed : %v", emails.RequestAccessToCLAManagersTemplateName, err) + return + } + + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -829,30 +939,18 @@ additional CLA Manager. } } -func sendRequestApprovedEmailToCLAManagers(companyModel *models.Company, claGroupModel *models.ClaGroup, requesterName, requesterEmail, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName +func sendRequestApprovedEmailToCLAManagers(emailSvc emails.EmailTemplateService, emailParams emails.RequestApprovedToCLAManagersTemplateParams, claGroupModel *models.ClaGroup) { projectName := claGroupModel.ProjectName // subject string, body string, recipients []string subject := fmt.Sprintf("EasyCLA: CLA Manager Access Approval Notice for %s", projectName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-The following user has been approved as a CLA Manager from %s for the project %s. This means that they can now -maintain the list of employees allowed to contribute to %s on behalf of your company, as well as view and manage the -list of company’s CLA Managers for %s.
-Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-You have now been approved as a CLA Manager from %s for the project %s. This means that you can now maintain the -list of employees allowed to contribute to %s on behalf of your company, as well as view and manage the list of your -company’s CLA Managers for %s.
-To get started, please log into the EasyCLA Corporate Console, and select your -company and then the project %s. From here you will be able to edit the list of approved employees and CLA Managers.
-%s -%s`, - requesterName, projectName, - companyName, projectName, projectName, projectName, - utils.GetCorporateURL(claGroupModel.Version == utils.V2), projectName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderRequestApprovedToRequesterTemplate(emailSvc, claGroupModel.Version, claGroupModel.ProjectExternalID, emailParams) + if err != nil { + log.Warnf("email template : %s failed rendering : %s", emails.RequestApprovedToRequesterTemplateName, err) + return + } + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -890,29 +977,20 @@ company and then the project %s. From here you will be able to edit the list of } } -func sendRequestDeniedEmailToCLAManagers(companyModel *models.Company, claGroupModel *models.ClaGroup, requesterName, requesterEmail, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName +func sendRequestDeniedEmailToCLAManagers(emailSvc emails.EmailTemplateService, emailParams emails.RequestDeniedToCLAManagersTemplateParams, claGroupModel *models.ClaGroup) { projectName := claGroupModel.ProjectName // subject string, body string, recipients []string subject := fmt.Sprintf("EasyCLA: CLA Manager Access Denied Notice for %s", projectName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-The following user has been denied as a CLA Manager from %s for the project %s. This means that they will not -be able to maintain the list of employees allowed to contribute to %s on behalf of your company.
-Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-You have been denied as a CLA Manager from %s for the project %s. This means that you can not maintain the -list of employees allowed to contribute to %s on behalf of your company.
-%s -%s`, - requesterName, projectName, - companyName, projectName, projectName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderRequestDeniedToRequesterTemplate(emailSvc, claGroupModel.Version, claGroupModel.ProjectExternalID, emails.RequestDeniedToRequesterTemplateParams{ + CommonEmailParams: emailParams, + }) + if err != nil { + log.Warnf("email template rendering %s failed : %v", emails.RequestDeniedToRequesterTemplateName, err) + return + } + + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { log.Debugf("sent email with subject: %s to recipients: %+v", subject, recipients) } } + +// ToAuthUser converts a legacy v1 CLA user to a v2 platform auth user +func ToAuthUser(claUser *user.CLAUser) *auth.User { + return &auth.User{ + UserName: claUser.LFUsername, + Email: claUser.LFEmail, + ACL: auth.ACL{}, + } +} diff --git a/cla-backend-go/cla_manager/models.go b/cla-backend-go/cla_manager/models.go index ff5e3a756..09e9fdb01 100644 --- a/cla-backend-go/cla_manager/models.go +++ b/cla-backend-go/cla_manager/models.go @@ -3,7 +3,7 @@ package cla_manager -import "github.com/communitybridge/easycla/cla-backend-go/gen/models" +import "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" // CLAManagerRequests data model type CLAManagerRequests struct { diff --git a/cla-backend-go/cla_manager/repository.go b/cla-backend-go/cla_manager/repository.go index 92fa3355d..6fe2d7088 100644 --- a/cla-backend-go/cla_manager/repository.go +++ b/cla-backend-go/cla_manager/repository.go @@ -6,7 +6,7 @@ package cla_manager import ( "fmt" - "github.com/communitybridge/easycla/cla-backend-go/project" + "github.com/linuxfoundation/easycla/cla-backend-go/project/models" "github.com/sirupsen/logrus" @@ -17,9 +17,9 @@ import ( "github.com/aws/aws-sdk-go/service/dynamodb" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" "github.com/aws/aws-sdk-go/service/dynamodb/expression" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/utils" "github.com/gofrs/uuid" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" ) // IRepository interface methods @@ -29,7 +29,7 @@ type IRepository interface { //nolint GetRequestsByUserID(companyID, projectID, userID string) (*CLAManagerRequests, error) GetRequest(requestID string) (*CLAManagerRequest, error) GetRequestsByCLAGroup(claGroupID string) ([]CLAManagerRequest, error) - UpdateRequestsByCLAGroup(model *project.DBProjectModel) error + UpdateRequestsByCLAGroup(model *models.DBProjectModel) error ApproveRequest(companyID, projectID, requestID string) (*CLAManagerRequest, error) DenyRequest(companyID, projectID, requestID string) (*CLAManagerRequest, error) @@ -480,7 +480,7 @@ func (repo repository) GetRequestsByCLAGroup(claGroupID string) ([]CLAManagerReq } // UpdateRequestsByCLAGroup handles updating the existing requests in our table based on the modified/updated CLA Group -func (repo repository) UpdateRequestsByCLAGroup(model *project.DBProjectModel) error { +func (repo repository) UpdateRequestsByCLAGroup(model *models.DBProjectModel) error { f := logrus.Fields{ "functionName": "UpdateRequestsByCLAGroup", "claGroupID": model.ProjectID, diff --git a/cla-backend-go/cla_manager/service.go b/cla-backend-go/cla_manager/service.go index fcfc355d1..fd96ac8cb 100644 --- a/cla-backend-go/cla_manager/service.go +++ b/cla-backend-go/cla_manager/service.go @@ -7,17 +7,24 @@ import ( "context" "fmt" + service2 "github.com/linuxfoundation/easycla/cla-backend-go/project/service" + + "github.com/LF-Engineering/lfx-kit/auth" + "github.com/sirupsen/logrus" + + "github.com/linuxfoundation/easycla/cla-backend-go/emails" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/aws/aws-sdk-go/aws" - "github.com/communitybridge/easycla/cla-backend-go/company" - "github.com/communitybridge/easycla/cla-backend-go/events" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" - sigAPI "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations/signatures" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/project" - "github.com/communitybridge/easycla/cla-backend-go/signatures" - "github.com/communitybridge/easycla/cla-backend-go/users" - "github.com/communitybridge/easycla/cla-backend-go/utils" - v2UserService "github.com/communitybridge/easycla/cla-backend-go/v2/user-service" + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + sigAPI "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/signatures" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + v2UserService "github.com/linuxfoundation/easycla/cla-backend-go/v2/user-service" ) // IService interface defining the functions for the company service @@ -32,30 +39,34 @@ type IService interface { PendingRequest(companyID, claGroupID, requestID string) (*models.ClaManagerRequest, error) DeleteRequest(requestID string) error - AddClaManager(ctx context.Context, companyID string, claGroupID string, LFID string) (*models.Signature, error) - RemoveClaManager(ctx context.Context, companyID string, claGroupID string, LFID string) (*models.Signature, error) + AddClaManager(ctx context.Context, authUser *auth.User, companyID string, claGroupID string, LFID string, projectSFID string) (*models.Signature, error) + RemoveClaManager(ctx context.Context, authUser *auth.User, companyID string, claGroupID string, LFID string, projectSFName string) (*models.Signature, error) } type service struct { - repo IRepository - companyService company.IService - projectService project.Service - usersService users.Service - sigService signatures.SignatureService - eventsService events.Service - corporateConsoleURL string + repo IRepository + projectClaRepository projects_cla_groups.Repository + companyService company.IService + projectService service2.Service + usersService users.Service + sigService signatures.SignatureService + eventsService events.Service + emailTemplateService emails.EmailTemplateService + corporateConsoleURL string } // NewService creates a new service object -func NewService(repo IRepository, companyService company.IService, projectService project.Service, usersService users.Service, sigService signatures.SignatureService, eventsService events.Service, corporateConsoleURL string) IService { +func NewService(repo IRepository, projectClaRepository projects_cla_groups.Repository, companyService company.IService, projectService service2.Service, usersService users.Service, sigService signatures.SignatureService, eventsService events.Service, emailTemplateService emails.EmailTemplateService, corporateConsoleURL string) IService { return service{ - repo: repo, - companyService: companyService, - projectService: projectService, - usersService: usersService, - sigService: sigService, - eventsService: eventsService, - corporateConsoleURL: corporateConsoleURL, + repo: repo, + projectClaRepository: projectClaRepository, + companyService: companyService, + projectService: projectService, + usersService: usersService, + sigService: sigService, + eventsService: eventsService, + emailTemplateService: emailTemplateService, + corporateConsoleURL: corporateConsoleURL, } } @@ -181,9 +192,18 @@ func (s service) DeleteRequest(requestID string) error { return nil } -// AddClaManager Adds LFID to Signature Access Control List list -func (s service) AddClaManager(ctx context.Context, companyID string, claGroupID string, LFID string) (*models.Signature, error) { +// AddClaManager Adds LFID to Signature Access Control list +func (s service) AddClaManager(ctx context.Context, authUser *auth.User, companyID string, claGroupID string, LFID string, projectSFID string) (*models.Signature, error) { + f := logrus.Fields{ + "functionName": "v1.cla_manager.AddClaManager", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "companyID": companyID, + "claGroupID": claGroupID, + "LFID": LFID, + "projectSFID": projectSFID, + } + var projectSFName string userModel, userErr := s.usersService.GetUserByLFUserName(LFID) if userErr != nil || userModel == nil { return nil, userErr @@ -209,11 +229,11 @@ func (s service) AddClaManager(ctx context.Context, companyID string, claGroupID claManagers := sigModel.SignatureACL - log.Debugf("Got Company signatures - Company: %s , Project: %s , signatureID: %s ", + log.WithFields(f).Debugf("Got Company signatures - Company: %s , Project: %s , signatureID: %s ", companyID, claGroupID, sigModel.SignatureID) // Update the signature ACL - addedSignature, aclErr := s.sigService.AddCLAManager(ctx, sigModel.SignatureID.String(), LFID) + addedSignature, aclErr := s.sigService.AddCLAManager(ctx, sigModel.SignatureID, LFID) if aclErr != nil { return nil, aclErr } @@ -221,34 +241,47 @@ func (s service) AddClaManager(ctx context.Context, companyID string, claGroupID // Update the company ACL record in EasyCLA companyACLError := s.companyService.AddUserToCompanyAccessList(ctx, companyID, LFID) if companyACLError != nil { - log.Warnf("AddCLAManager- Unable to add user to company ACL, companyID: %s, user: %s, error: %+v", companyID, LFID, companyACLError) + log.WithFields(f).Warnf("AddCLAManager- Unable to add user to company ACL, companyID: %s, user: %s, error: %+v", companyID, LFID, companyACLError) return nil, companyACLError } // Notify CLA Managers - send email to each manager for _, manager := range claManagers { - sendClaManagerAddedEmailToCLAManagers(companyModel, claGroupModel, userModel.Username, userModel.LfEmail, - manager.Username, manager.LfEmail) + sendClaManagerAddedEmailToCLAManagers(s.emailTemplateService, emails.ClaManagerAddedToCLAManagersTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: manager.Username, + RecipientAddress: manager.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, + Name: userModel.Username, + Email: userModel.LfEmail.String(), + ProjectSFID: projectSFID, + }, claGroupModel) } // Notify the added user - sendClaManagerAddedEmailToUser(companyModel, claGroupModel, userModel.Username, userModel.LfEmail) + s.sendClaManagerAddedEmailToUser(s.emailTemplateService, emails.CommonEmailParams{ + RecipientName: userModel.Username, + RecipientAddress: userModel.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, claGroupModel, projectSFID) // Send an event - s.eventsService.LogEvent(&events.LogEventArgs{ - EventType: events.ClaManagerCreated, - ProjectID: claGroupID, - ClaGroupModel: claGroupModel, - CompanyID: companyID, - CompanyModel: companyModel, - LfUsername: LFID, - UserID: LFID, - UserModel: userModel, - ExternalProjectID: claGroupModel.ProjectExternalID, + s.eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ + EventType: events.ClaManagerCreated, + UserName: authUser.UserName, + LfUsername: authUser.UserName, + CLAGroupID: claGroupID, + CLAGroupName: claGroupModel.ProjectName, + ClaGroupModel: claGroupModel, + ProjectID: claGroupModel.ProjectExternalID, + ProjectSFID: claGroupModel.ProjectExternalID, + CompanyID: companyID, + CompanyModel: companyModel, EventData: &events.CLAManagerCreatedEventData{ CompanyName: companyModel.CompanyName, - ProjectName: claGroupModel.ProjectName, + ProjectName: projectSFName, UserName: userModel.Username, - UserEmail: userModel.LfEmail, + UserEmail: userModel.LfEmail.String(), UserLFID: userModel.LfUsername, }, }) @@ -280,8 +313,17 @@ func (s service) getCompanySignature(ctx context.Context, companyID string, claG } // RemoveClaManager removes lfid from signature acl with given company and project -func (s service) RemoveClaManager(ctx context.Context, companyID string, claGroupID string, LFID string) (*models.Signature, error) { +func (s service) RemoveClaManager(ctx context.Context, authUser *auth.User, companyID string, claGroupID string, LFID string, projectSFID string) (*models.Signature, error) { + f := logrus.Fields{ + "functionName": "v1.cla_manager.RemoveClaManager", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + "claGroupID": claGroupID, + "LFID": LFID, + "companyID": companyID, + } + + var projectSFName string userModel, userErr := s.usersService.GetUserByLFUserName(LFID) if userErr != nil || userModel == nil { return nil, userErr @@ -303,10 +345,17 @@ func (s service) RemoveClaManager(ctx context.Context, companyID string, claGrou return nil, sigErr } + if len(sigModel.SignatureACL) <= 1 { + // Can't delete the only remaining CLA Manager.... + return nil, &utils.CLAManagerError{ + Message: "unable to remove the only remaining CLA Manager - signed CLAs must have at least one CLA Manager", + } + } + // Update the signature ACL - updatedSignature, aclErr := s.sigService.RemoveCLAManager(ctx, sigModel.SignatureID.String(), LFID) + updatedSignature, aclErr := s.sigService.RemoveCLAManager(ctx, sigModel.SignatureID, LFID) if aclErr != nil || updatedSignature == nil { - log.Warnf("remove CLA Manager returned an error or empty signature model using Signature ID: %s, error: %+v", + log.WithFields(f).Warnf("remove CLA Manager returned an error or empty signature model using Signature ID: %s, error: %+v", sigModel.SignatureID, sigErr) return nil, aclErr } @@ -319,29 +368,41 @@ func (s service) RemoveClaManager(ctx context.Context, companyID string, claGrou claManagers := sigModel.SignatureACL // Notify CLA Managers - send email to each manager for _, manager := range claManagers { - sendClaManagerDeleteEmailToCLAManagers(companyModel, claGroupModel, userModel.LfUsername, userModel.LfEmail, - manager.Username, manager.LfEmail) + s.sendClaManagerDeleteEmailToCLAManagers(s.emailTemplateService, emails.ClaManagerDeletedToCLAManagersTemplateParams{ + CommonEmailParams: emails.CommonEmailParams{ + RecipientName: manager.Username, + RecipientAddress: manager.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, + Name: userModel.LfUsername, + Email: userModel.LfEmail.String(), + }, claGroupModel) } // Notify the removed manager - sendRemovedClaManagerEmailToRecipient(companyModel, claGroupModel, userModel.LfUsername, userModel.LfEmail, claManagers) + sendRemovedClaManagerEmailToRecipient(s.emailTemplateService, emails.CommonEmailParams{ + RecipientName: userModel.LfUsername, + RecipientAddress: userModel.LfEmail.String(), + CompanyName: companyModel.CompanyName, + }, claGroupModel, claManagers) // Send an event s.eventsService.LogEvent(&events.LogEventArgs{ - EventType: events.ClaManagerDeleted, - ProjectID: claGroupID, - ClaGroupModel: claGroupModel, - CompanyID: companyID, - CompanyModel: companyModel, - LfUsername: userModel.LfUsername, - UserID: LFID, - UserModel: userModel, - ExternalProjectID: claGroupModel.ProjectExternalID, + EventType: events.ClaManagerDeleted, + LfUsername: authUser.UserName, + UserName: authUser.UserName, + CLAGroupID: claGroupID, + CLAGroupName: claGroupModel.ProjectName, + ClaGroupModel: claGroupModel, + ProjectID: claGroupModel.ProjectExternalID, + ProjectSFID: claGroupModel.ProjectExternalID, + CompanyID: companyID, + CompanyModel: companyModel, EventData: &events.CLAManagerDeletedEventData{ CompanyName: companyModel.CompanyName, - ProjectName: claGroupModel.ProjectName, + ProjectName: projectSFName, UserName: userModel.LfUsername, - UserEmail: userModel.LfEmail, + UserEmail: userModel.LfEmail.String(), UserLFID: LFID, }, }) @@ -349,29 +410,68 @@ func (s service) RemoveClaManager(ctx context.Context, companyID string, claGrou return updatedSignature, nil } -func sendClaManagerAddedEmailToUser(companyModel *models.Company, claGroupModel *models.ClaGroup, requesterName, requesterEmail string) { - companyName := companyModel.CompanyName - projectName := claGroupModel.ProjectName +type ProjectDetails struct { + ProjectName string + ProjectSFID []string +} + +// func (s service) getProjectDetails(ctx context.Context, claGroupModel *models.ClaGroup) ProjectDetails { +// f := logrus.Fields{ +// "functionName": "v1.cla_manager.getProjectDetails", +// utils.XREQUESTID: ctx.Value(utils.XREQUESTID), +// "claGroupID": claGroupModel.ProjectID, +// } +// projectSFIDs := make([]string, 0) + +// projectDetails := ProjectDetails{ +// ProjectName: claGroupModel.ProjectName, +// } +// signedAtFoundation := false +// var err error + +// pcg, pcgErr := s.projectClaRepository.GetCLAGroup(ctx, claGroupModel.ProjectID) +// if pcgErr != nil { +// log.WithFields(f).WithError(err).Debug("unable too get pcg record") +// } + +// signedAtFoundation, err = s.projectClaRepository.SignedAtFoundation(ctx, claGroupModel.ProjectID) +// if err != nil { +// log.WithFields(f).WithError(err).Debug("unable to get status of cla signed at foundation") +// } + +// if signedAtFoundation && pcg != nil && err != nil { +// log.WithFields(f).Debug("cla group is signed at foundation level...") +// projectDetails.ProjectName = pcg.FoundationName +// projectSFIDs = append(projectSFIDs, pcg.FoundationSFID) + +// } else { +// log.WithFields(f).Debug("cla group is signed at project level ...") + +// } + +// projectDetails.ProjectSFID = projectSFIDs +// return projectDetails +// } + +func (s service) sendClaManagerAddedEmailToUser(emailSvc emails.EmailTemplateService, emailParams emails.CommonEmailParams, claGroupModel *models.ClaGroup, projectSFID string) { + f := logrus.Fields{ + "functionName": "sendClaManagerAddedEmailToUser", + "projectSFID": projectSFID, + } + log.WithFields(f).Info("Sending email to user") // subject string, body string, recipients []string - subject := fmt.Sprintf("EasyCLA: Added as CLA Manager for Project :%s", projectName) - recipients := []string{requesterEmail} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-You have been added as a CLA Manager from %s for the project %s. This means that you can now maintain the -list of employees allowed to contribute to %s on behalf of your company, as well as view and manage the list of your -company’s CLA Managers for %s.
-To get started, please log into the EasyCLA Corporate Console, and select your -company and then the project %s. From here you will be able to edit the list of approved employees and CLA Managers.
-%s -%s`, - requesterName, projectName, - companyName, projectName, projectName, projectName, - utils.GetCorporateURL(claGroupModel.Version == utils.V2), projectName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + subject := fmt.Sprintf("EasyCLA: Added as CLA Manager for Project :%s", claGroupModel.ProjectName) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderClaManagerAddedEToUserTemplate(emailSvc, claGroupModel.Version, projectSFID, emails.ClaManagerAddedEToUserTemplateParams{ + CommonEmailParams: emailParams, + }) + if err != nil { + log.Warnf("email template render : %s failed : %v", emails.ClaManagerAddedEToUserTemplateName, err) + return + } + + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -379,30 +479,19 @@ company and then the project %s. From here you will be able to edit the list of } } -func sendClaManagerAddedEmailToCLAManagers(companyModel *models.Company, claGroupModel *models.ClaGroup, name, email, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName +func sendClaManagerAddedEmailToCLAManagers(emailSvc emails.EmailTemplateService, emailParams emails.ClaManagerAddedToCLAManagersTemplateParams, claGroupModel *models.ClaGroup) { projectName := claGroupModel.ProjectName // subject string, body string, recipients []string subject := fmt.Sprintf("EasyCLA: CLA Manager Added Notice for %s", projectName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-The following user has been added as a CLA Manager from %s for the project %s. This means that they can now -maintain the list of employees allowed to contribute to %s on behalf of your company, as well as view and manage the -list of company’s CLA Managers for %s.
-Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-You have been removed as a CLA Manager from %s for the project %s.
-If you have further questions about this, please contact one of the existing managers from -%s:
-%s -%s -%s`, - recipientName, projectName, companyName, projectName, companyName, companyManagerText, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderRemovedCLAManagerTemplate( + emailSvc, + claGroupModel.Version, + emails.RemovedCLAManagerTemplateParams{ + CommonEmailParams: emailParams, + CLAManagers: emailCLAManagerParams, + CLAGroupTemplateParams: emails.CLAGroupTemplateParams{ + CLAGroupName: projectName, + }, + }) + + if err != nil { + log.Warnf("rendering the email content failed for : %s", emails.RemovedCLAManagerTemplateName) + return + } + + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { @@ -479,24 +572,19 @@ func sendRemovedClaManagerEmailToRecipient(companyModel *models.Company, claGrou } } -func sendClaManagerDeleteEmailToCLAManagers(companyModel *models.Company, claGroupModel *models.ClaGroup, name, email, recipientName, recipientAddress string) { - companyName := companyModel.CompanyName - projectName := claGroupModel.ProjectName +func (s service) sendClaManagerDeleteEmailToCLAManagers(emailSvc emails.EmailTemplateService, emailParams emails.ClaManagerDeletedToCLAManagersTemplateParams, claGroupModel *models.ClaGroup) { // subject string, body string, recipients []string - subject := fmt.Sprintf("EasyCLA: CLA Manager Removed Notice for %s", projectName) - recipients := []string{recipientAddress} - body := fmt.Sprintf(` -Hello %s,
-This is a notification email from EasyCLA regarding the project %s.
-%s(%s) has been removed as a CLA Manager from %s for the project %s.
-%s -%s -`, - recipientName, projectName, name, email, companyName, projectName, - utils.GetEmailHelpContent(claGroupModel.Version == utils.V2), utils.GetEmailSignOffContent()) - - err := utils.SendEmail(subject, body, recipients) + subject := fmt.Sprintf("EasyCLA: CLA Manager Removed Notice for %s", claGroupModel.ProjectName) + recipients := []string{emailParams.RecipientAddress} + body, err := emails.RenderClaManagerDeletedToCLAManagersTemplate(emailSvc, claGroupModel.Version, claGroupModel.ProjectName) + + if err != nil { + log.Warnf("email template render : %s failed : %v", emails.ClaManagerDeletedToCLAManagersTemplateName, err) + return + } + + err = utils.SendEmail(subject, body, recipients) if err != nil { log.Warnf("problem sending email with subject: %s to recipients: %+v, error: %+v", subject, recipients, err) } else { diff --git a/cla-backend-go/cmd/dynamo_events_lambda/main.go b/cla-backend-go/cmd/dynamo_events_lambda/main.go index f9dba32de..91161ad53 100644 --- a/cla-backend-go/cmd/dynamo_events_lambda/main.go +++ b/cla-backend-go/cmd/dynamo_events_lambda/main.go @@ -6,48 +6,59 @@ package main import ( "context" "encoding/json" + "fmt" "os" - "github.com/communitybridge/easycla/cla-backend-go/github_organizations" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + "github.com/linuxfoundation/easycla/cla-backend-go/project/service" - "github.com/communitybridge/easycla/cla-backend-go/utils" + v2Repositories "github.com/linuxfoundation/easycla/cla-backend-go/v2/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/store" - "github.com/communitybridge/easycla/cla-backend-go/approval_list" - "github.com/communitybridge/easycla/cla-backend-go/cla_manager" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/gitlab_organizations" - "github.com/communitybridge/easycla/cla-backend-go/gerrits" - "github.com/communitybridge/easycla/cla-backend-go/project" - "github.com/communitybridge/easycla/cla-backend-go/repositories" + gitlab "github.com/linuxfoundation/easycla/cla-backend-go/gitlab_api" - acs_service "github.com/communitybridge/easycla/cla-backend-go/v2/acs-service" - organization_service "github.com/communitybridge/easycla/cla-backend-go/v2/organization-service" - project_service "github.com/communitybridge/easycla/cla-backend-go/v2/project-service" - user_service "github.com/communitybridge/easycla/cla-backend-go/v2/user-service" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" - "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" - "github.com/communitybridge/easycla/cla-backend-go/v2/dynamo_events" + "github.com/linuxfoundation/easycla/cla-backend-go/approval_list" + "github.com/linuxfoundation/easycla/cla-backend-go/cla_manager" - "github.com/communitybridge/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" - "github.com/communitybridge/easycla/cla-backend-go/company" - "github.com/communitybridge/easycla/cla-backend-go/github" - v2Company "github.com/communitybridge/easycla/cla-backend-go/v2/company" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" + project_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service" + user_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/user-service" - claevents "github.com/communitybridge/easycla/cla-backend-go/events" - "github.com/communitybridge/easycla/cla-backend-go/user" - "github.com/communitybridge/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/dynamo_events" + + "github.com/linuxfoundation/easycla/cla-backend-go/token" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/github" + v2Company "github.com/linuxfoundation/easycla/cla-backend-go/v2/company" + + claevents "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/user" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" "github.com/aws/aws-lambda-go/lambda" - "github.com/communitybridge/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/config" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" ) var ( @@ -80,44 +91,54 @@ func init() { usersRepo := users.NewRepository(awsSession, stage) userRepo := user.NewDynamoRepository(awsSession, stage) companyRepo := company.NewRepository(awsSession, stage) - signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo) projectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) + v2Repository := v2Repositories.NewRepository(awsSession, stage) repositoriesRepo := repositories.NewRepository(awsSession, stage) gerritRepo := gerrits.NewRepository(awsSession, stage) - projectRepo := project.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) + projectRepo := repository.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) eventsRepo := claevents.NewRepository(awsSession, stage) claManagerRequestsRepo := cla_manager.NewRepository(awsSession, stage) approvalListRequestsRepo := approval_list.NewRepository(awsSession, stage) githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) + gitlabOrganizationRepo := gitlab_organizations.NewRepository(awsSession, stage) + storeRepo := store.NewRepository(awsSession, stage) + approvalsTableName := fmt.Sprintf("cla-%s-approvals", stage) + approvalRepo := approvals.NewRepository(stage, awsSession, approvalsTableName) token.Init(configFile.Auth0Platform.ClientID, configFile.Auth0Platform.ClientSecret, configFile.Auth0Platform.URL, configFile.Auth0Platform.Audience) - github.Init(configFile.Github.AppID, configFile.Github.AppPrivateKey, configFile.Github.AccessToken) + github.Init(configFile.GitHub.AppID, configFile.GitHub.AppPrivateKey, configFile.GitHub.AccessToken) + // initialize gitlab + gitlabApp := gitlab.Init(configFile.Gitlab.AppClientID, configFile.Gitlab.AppClientSecret, configFile.Gitlab.AppPrivateKey) user_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) project_service.InitClient(configFile.APIGatewayURL) githubOrganizationsService := github_organizations.NewService(githubOrganizationsRepo, repositoriesRepo, projectClaGroupRepo) repositoriesService := repositories.NewService(repositoriesRepo, githubOrganizationsRepo, projectClaGroupRepo) - gerritService := gerrits.NewService(gerritRepo, &gerrits.LFGroup{ - LfBaseURL: configFile.LFGroup.ClientURL, - ClientID: configFile.LFGroup.ClientID, - ClientSecret: configFile.LFGroup.ClientSecret, - RefreshToken: configFile.LFGroup.RefreshToken, - }) + + gerritService := gerrits.NewService(gerritRepo) // Services - projectService := project.NewService(projectRepo, repositoriesRepo, gerritRepo, projectClaGroupRepo, usersRepo) + projectService := service.NewService(projectRepo, repositoriesRepo, gerritRepo, projectClaGroupRepo, usersRepo) type combinedRepo struct { users.UserRepository company.IRepository - project.ProjectRepository + repository.ProjectRepository + projects_cla_groups.Repository } + eventsService := claevents.NewService(eventsRepo, combinedRepo{ usersRepo, companyRepo, projectRepo, + projectClaGroupRepo, }) + usersService := users.NewService(usersRepo, eventsService) - companyService := company.NewService(companyRepo, configFile.CorporateConsoleURL, userRepo, usersService) + signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, repositoriesRepo, githubOrganizationsRepo, gerritService, approvalRepo) + v2RepositoryService := v2Repositories.NewService(repositoriesRepo, v2Repository, projectClaGroupRepo, githubOrganizationsRepo, gitlabOrganizationRepo, eventsService) + gitlabOrgService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoryService, projectClaGroupRepo, storeRepo, usersService, signaturesRepo, companyRepo) + + companyService := company.NewService(companyRepo, configFile.CorporateConsoleV1URL, userRepo, usersService) v2CompanyService := v2Company.NewService(companyService, signaturesRepo, projectRepo, usersRepo, companyRepo, projectClaGroupRepo, eventsService) organization_service.InitClient(configFile.APIGatewayURL, eventsService) acs_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) @@ -129,12 +150,17 @@ func init() { projectClaGroupRepo, eventsRepo, projectRepo, + gitlabOrganizationRepo, + v2Repository, projectService, githubOrganizationsService, repositoriesService, gerritService, claManagerRequestsRepo, - approvalListRequestsRepo) + approvalListRequestsRepo, + gitlabApp, + gitlabOrgService, + ) } func handler(ctx context.Context, event events.DynamoDBEvent) { diff --git a/cla-backend-go/cmd/functional_tests/approval_list/approval_list.go b/cla-backend-go/cmd/functional_tests/approval_list/approval_list.go index f63879840..031cc13d4 100644 --- a/cla-backend-go/cmd/functional_tests/approval_list/approval_list.go +++ b/cla-backend-go/cmd/functional_tests/approval_list/approval_list.go @@ -8,9 +8,9 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/gen/v2/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" "github.com/verdverm/frisby" ) diff --git a/cla-backend-go/cmd/functional_tests/cla_group/cla_group.go b/cla-backend-go/cmd/functional_tests/cla_group/cla_group.go index 0c07444ab..11013ac3f 100644 --- a/cla-backend-go/cmd/functional_tests/cla_group/cla_group.go +++ b/cla-backend-go/cmd/functional_tests/cla_group/cla_group.go @@ -8,8 +8,8 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/gen/v2/models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" "github.com/verdverm/frisby" ) diff --git a/cla-backend-go/cmd/functional_tests/cla_manager/cla_manager.go b/cla-backend-go/cmd/functional_tests/cla_manager/cla_manager.go index a7edc9b98..3fece57b4 100644 --- a/cla-backend-go/cmd/functional_tests/cla_manager/cla_manager.go +++ b/cla-backend-go/cmd/functional_tests/cla_manager/cla_manager.go @@ -8,15 +8,15 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" "github.com/verdverm/frisby" ) var ( claManagerToken string claProspectiveManagerToken string - claManagerCreateRequestID string = "no-set" + claManagerCreateRequestID = "no-set" ) const ( @@ -281,7 +281,7 @@ func (t *TestBehaviour) RunGetCLAManagerRequests() { if unmarshallErr != nil { F.AddError(unmarshallErr.Error()) } - if requests.Requests == nil || len(requests.Requests) == 0 { + if len(requests.Requests) == 0 { F.AddError("GET CLA Manager Requests - Expecting at least one request in response") } var containsEntry = false diff --git a/cla-backend-go/cmd/functional_tests/company/company.go b/cla-backend-go/cmd/functional_tests/company/company.go index dbc507df9..760c88d4d 100644 --- a/cla-backend-go/cmd/functional_tests/company/company.go +++ b/cla-backend-go/cmd/functional_tests/company/company.go @@ -9,11 +9,11 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/gen/v2/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" "github.com/LF-Engineering/lfx-kit/auth" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/verdverm/frisby" ) diff --git a/cla-backend-go/cmd/functional_tests/health/health.go b/cla-backend-go/cmd/functional_tests/health/health.go index 8fe3d7ec7..0ff940c80 100644 --- a/cla-backend-go/cmd/functional_tests/health/health.go +++ b/cla-backend-go/cmd/functional_tests/health/health.go @@ -8,8 +8,8 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" "github.com/verdverm/frisby" ) diff --git a/cla-backend-go/cmd/functional_tests/main.go b/cla-backend-go/cmd/functional_tests/main.go index 84d17da2f..3d5342b24 100644 --- a/cla-backend-go/cmd/functional_tests/main.go +++ b/cla-backend-go/cmd/functional_tests/main.go @@ -6,18 +6,18 @@ package main import ( "os" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/approval_list" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/cla_group" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/cla_manager" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/company" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/health" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/signatures" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/template" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/approval_list" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/cla_group" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/cla_manager" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/company" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/health" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/template" - "github.com/communitybridge/easycla/cla-backend-go/cmd/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/repositories" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/verdverm/frisby" ) diff --git a/cla-backend-go/cmd/functional_tests/org_service/org_service.go b/cla-backend-go/cmd/functional_tests/org_service/org_service.go index d05685fe3..d3cd1c65b 100644 --- a/cla-backend-go/cmd/functional_tests/org_service/org_service.go +++ b/cla-backend-go/cmd/functional_tests/org_service/org_service.go @@ -6,19 +6,20 @@ package org_service import ( "context" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/company" - "github.com/communitybridge/easycla/cla-backend-go/config" - "github.com/communitybridge/easycla/cla-backend-go/events" - "github.com/communitybridge/easycla/cla-backend-go/gerrits" - ini "github.com/communitybridge/easycla/cla-backend-go/init" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/project" - "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/repositories" - "github.com/communitybridge/easycla/cla-backend-go/users" - acs_service "github.com/communitybridge/easycla/cla-backend-go/v2/acs-service" - organization_service "github.com/communitybridge/easycla/cla-backend-go/v2/organization-service" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + ini "github.com/linuxfoundation/easycla/cla-backend-go/init" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" "github.com/spf13/viper" ) @@ -56,21 +57,25 @@ func (t *TestBehaviour) RunIsUserHaveRoleScope() { type combinedRepo struct { users.UserRepository company.IRepository - project.ProjectRepository + repository.ProjectRepository + projects_cla_groups.Repository } + eventsRepo := events.NewRepository(awsSession, stage) usersRepo := users.NewRepository(awsSession, stage) companyRepo := company.NewRepository(awsSession, stage) repositoriesRepo := repositories.NewRepository(awsSession, stage) gerritRepo := gerrits.NewRepository(awsSession, stage) projectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) - projectRepo := project.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) + projectRepo := repository.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) eventsService := events.NewService(eventsRepo, combinedRepo{ usersRepo, companyRepo, projectRepo, + projectClaGroupRepo, }) + organization_service.InitClient(configFile.APIGatewayURL, eventsService) acs_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) acsClient := acs_service.GetClient() diff --git a/cla-backend-go/cmd/functional_tests/signatures/signatures.go b/cla-backend-go/cmd/functional_tests/signatures/signatures.go index 340324efd..9811da338 100644 --- a/cla-backend-go/cmd/functional_tests/signatures/signatures.go +++ b/cla-backend-go/cmd/functional_tests/signatures/signatures.go @@ -8,8 +8,8 @@ import ( "fmt" "reflect" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" "github.com/verdverm/frisby" ) @@ -174,7 +174,7 @@ func (t *TestBehaviour) RunGetCompanySignatures() { if unmarshallErr != nil { F.AddError(unmarshallErr.Error()) } - if signatures.Signatures == nil || len(signatures.Signatures) == 0 { + if len(signatures.Signatures) == 0 { F.AddError("Signatures - Get Company Signatures - Google - Expecting at least one signature in response") } for _, sig := range signatures.Signatures { diff --git a/cla-backend-go/cmd/functional_tests/signatures/xacl.go b/cla-backend-go/cmd/functional_tests/signatures/xacl.go index a10124580..3ba7b2d42 100644 --- a/cla-backend-go/cmd/functional_tests/signatures/xacl.go +++ b/cla-backend-go/cmd/functional_tests/signatures/xacl.go @@ -8,7 +8,7 @@ import ( "encoding/json" "github.com/LF-Engineering/lfx-kit/auth" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" ) // GetXACLGoogle returns the X-ACL entry for this company diff --git a/cla-backend-go/cmd/functional_tests/template/template.go b/cla-backend-go/cmd/functional_tests/template/template.go index 8a37c09d6..cf84adef8 100644 --- a/cla-backend-go/cmd/functional_tests/template/template.go +++ b/cla-backend-go/cmd/functional_tests/template/template.go @@ -7,7 +7,7 @@ import ( "bytes" "fmt" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" "github.com/pdfcpu/pdfcpu/pkg/api" "github.com/pdfcpu/pdfcpu/pkg/pdfcpu" "github.com/verdverm/frisby" diff --git a/cla-backend-go/cmd/generate_compound_attribute/main.go b/cla-backend-go/cmd/generate_compound_attribute/main.go new file mode 100644 index 000000000..a038e7a25 --- /dev/null +++ b/cla-backend-go/cmd/generate_compound_attribute/main.go @@ -0,0 +1,160 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" +) + +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var events EventInterface +var stage string + +const ( + tableName = "cla-%s-events" + attr1 = "event_company_sfid" + attr2 = "event_cla_group_id" + newAttr = "company_sfid_cla_group_id" + sep = "#" +) + +type EventInterface interface { + FetchAndUpdateDocuments(ctx context.Context) error + InsertCompoundAttribute(ctx context.Context, event EventModel) error +} + +type Config struct { + tableName string + dynamoDBClient *dynamodb.DynamoDB + stage string +} + +type EventModel struct { + EventID string `json:"event_id"` + EventCompanySFID string `json:"event_company_sfid"` + EventCLAGroupID string `json:"event_cla_group_id"` +} + +func (c Config) FetchAndUpdateDocuments(ctx context.Context) error { + builder := expression.NewBuilder() + filter := expression.Name(attr1).AttributeExists().And(expression.Name(attr2).AttributeExists()).And(expression.Name(newAttr).AttributeNotExists()) + builder = builder.WithFilter(filter) + expr, err := builder.Build() + if err != nil { + log.Error("stage not set", err) + return err + } + var lastEvaluatedKey string + // Assemble the query input parameters + scanInput := &dynamodb.ScanInput{ + TableName: aws.String(c.tableName), + FilterExpression: expr.Filter(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + } + var total int + for ok := true; ok; ok = lastEvaluatedKey != "" { + results, err := c.dynamoDBClient.ScanWithContext(ctx, scanInput) + if err != nil { + log.Error("Found error on ScanWithContext", err) + return err + } + log.Debugf("Found ---> %d Items in a batch", len(results.Items)) + if len(results.Items) > 0 { + total += len(results.Items) + + var events []EventModel + err = dynamodbattribute.UnmarshalListOfMaps(results.Items, &events) + if err != nil { + log.Error("Found error on UnmarshalListOfMaps", err) + return err + } + for _, event := range events { + err = c.InsertCompoundAttribute(ctx, event) + if err != nil { + log.Error("Found error on InsertCompoundAttribute", err) + return err + } + } + log.Debugf("All items are updated of the batch %d", len(results.Items)) + } + + if results.LastEvaluatedKey["event_id"] != nil { + //log.Debugf("LastEvaluatedKey: %+v", result.LastEvaluatedKey["signature_id"]) + lastEvaluatedKey = *results.LastEvaluatedKey["event_id"].S + scanInput.ExclusiveStartKey = map[string]*dynamodb.AttributeValue{ + "event_id": { + S: aws.String(lastEvaluatedKey), + }, + } + } else { + lastEvaluatedKey = "" + } + } + fmt.Printf("total Items: %d", total) + return nil +} + +func (c Config) InsertCompoundAttribute(ctx context.Context, event EventModel) error { + updateExpression := expression.Set(expression.Name(newAttr), expression.Value(fmt.Sprintf("%s#%s", event.EventCompanySFID, event.EventCLAGroupID))) + expr, err := expression.NewBuilder().WithUpdate(updateExpression).Build() + if err != nil { + log.Error("Found error on NewBuilder", err) + return err + } + + _, err = c.dynamoDBClient.UpdateItemWithContext(ctx, &dynamodb.UpdateItemInput{ + TableName: aws.String(c.tableName), + Key: map[string]*dynamodb.AttributeValue{ + "event_id": { + S: aws.String(event.EventID), + }, + }, + UpdateExpression: expr.Update(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + }) + if err != nil { + log.Error("Found error on UpdateItemWithContext", err) + return err + } + log.Debugf("Updates event - %s", event.EventID) + return nil +} + +func NewRepository(awsSession *session.Session, stage string) EventInterface { + return &Config{ + dynamoDBClient: dynamodb.New(awsSession), + stage: stage, + tableName: fmt.Sprintf(tableName, stage), + } +} + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + log.Infof("STAGE set to %s\n", stage) + events = NewRepository(awsSession, stage) +} + +func main() { + log.Debugf("Getting events that should be updated...") + + context := context.Background() + err := events.FetchAndUpdateDocuments(context) + if err != nil { + panic(err) + } +} diff --git a/cla-backend-go/cmd/gitlab/api/main.go b/cla-backend-go/cmd/gitlab/api/main.go new file mode 100644 index 000000000..54b9b34f0 --- /dev/null +++ b/cla-backend-go/cmd/gitlab/api/main.go @@ -0,0 +1,117 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "flag" + "fmt" + "os" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/xanzy/go-gitlab" +) + +const ( + ProjectsURL = "https://gitlab.com/api/v4/projects" +) + +var state = flag.String("state", "failed", "the state of the MR to set") + +func main() { + flag.Parse() + + access_token := os.Getenv("GITLAB_ACCESS_TOKEN") + if access_token == "" { + log.Fatal("GITLAB_ACCESS_TOKEN is required") + } + + log.Infof("The gitlab access token is : %s", access_token) + + gitlabClient, err := gitlab.NewOAuthClient(access_token) + if err != nil { + log.Fatalf("creating client failed : %v", err) + } + + user, _, err := gitlabClient.Users.CurrentUser() + if err != nil { + log.Fatalf("fetching current user failed : %v", err) + } + + log.Infof("fetched current user : %s", user.Name) + + projects, _, err := gitlabClient.Projects.ListUserProjects(user.ID, &gitlab.ListProjectsOptions{}) + if err != nil { + log.Fatalf("listing projects failed : %v", err) + } + log.Printf("we fetched : %d projects for the account", len(projects)) + for _, p := range projects { + log.Println("Name : ", p.Name) + log.Println("ID: ", p.ID) + } + + projectID := 28118160 + commitSha := "f7036ab67a4e464e83e16af0b02d447c53fffa74" + + statuses, _, err := gitlabClient.Commits.GetCommitStatuses(projectID, commitSha, + &gitlab.GetCommitStatusesOptions{}) + if err != nil { + log.Fatalf("fetching commit statuses failed : %v", err) + } + + if len(statuses) == 0 { + log.Infof("no statuses found for commit sha") + setState := gitlab.Failed + if *state != string(gitlab.Failed) { + setState = gitlab.Success + } + + _, _, err = gitlabClient.Commits.SetCommitStatus(projectID, commitSha, &gitlab.SetCommitStatusOptions{ + State: setState, + Name: gitlab.String("easyCLA Bot"), + Description: gitlab.String(getDescription(setState)), + TargetURL: gitlab.String(getTargetURL("deniskurov@gmail.com")), + }) + if err != nil { + log.Fatalf("setting commit status for the sha failed : %v", err) + } + + statuses, _, err = gitlabClient.Commits.GetCommitStatuses(projectID, commitSha, + &gitlab.GetCommitStatusesOptions{}) + if err != nil { + log.Fatalf("fetching commit statuses failed : %v", err) + } + + } + + for _, status := range statuses { + log.Println("Status : ", status.Status) + if status.Status != *state { + log.Infof("setting state of commit sha to %s", *state) + _, _, err = gitlabClient.Commits.SetCommitStatus(projectID, commitSha, &gitlab.SetCommitStatusOptions{ + State: gitlab.BuildStateValue(*state), + Name: gitlab.String("easyCLA Bot"), + Description: gitlab.String(getDescription(gitlab.BuildStateValue(*state))), + TargetURL: gitlab.String(getTargetURL("deniskurov@gmail.com")), + }) + if err != nil { + log.Fatalf("setting commit status for the sha failed : %v", err) + } + } + log.Println("Status Name : ", status.Name) + log.Println("Status Description : ", status.Description) + log.Println("Status Author : ", status.Author.Name) + log.Println("Status Author Email : ", status.Author.Email) + } +} + +func getDescription(status gitlab.BuildStateValue) string { + if status == gitlab.Failed { + return "User hasn't signed CLA" + } + return "User signed CLA" +} + +func getTargetURL(email string) string { + return fmt.Sprintf("http://localhost:8080/gitlab/sign/%s", email) +} diff --git a/cla-backend-go/cmd/gitlab/auth/main.go b/cla-backend-go/cmd/gitlab/auth/main.go new file mode 100644 index 000000000..f6625373d --- /dev/null +++ b/cla-backend-go/cmd/gitlab/auth/main.go @@ -0,0 +1,319 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "strconv" + "sync" + + "github.com/gin-gonic/gin" + "github.com/go-resty/resty/v2" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/xanzy/go-gitlab" +) + +const ( + clientRedirectURI = "http://localhost:8080/gitlab/oauth/callback" + clientAppID = "18718b478096e6a257eda51414d0d446ad28866c15187aa765f602fe906d0b17" + clientAppSecret = "8dd14ace0eb0e4674b849b6fed4ce51bbcc456fc62d9149aff15353c1dda6327" +) + +const ( + hookURL = "https://4c1ba3f4f3c1.ngrok.io/gitlab/events" +) + +type OauthSuccessResponse struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` + ExpiresIn int `json:"expires_in"` + RefreshToken string `json:"refresh_token"` + CreatedAt int `json:"created_at"` +} + +var passingUsers = map[string]bool{ + "deniskurov@gmail.com": true, +} + +var passingUsersMutex sync.RWMutex // Protects passingUsers map + +func main() { + r := gin.Default() + r.GET("/gitlab/sign", func(c *gin.Context) { + email := c.Query("email") + if email == "" { + c.JSON(400, gin.H{ + "message": "email is required parameter", + }) + return + } + + projectID := c.Query("project_id") + if projectID == "" { + c.JSON(400, gin.H{ + "message": "projectID is required parameter", + }) + return + } + + lastCommitSha := c.Query("sha") + if lastCommitSha == "" { + c.JSON(400, gin.H{ + "message": "sha is required parameter", + }) + return + } + + projectIDInt, err := strconv.Atoi(projectID) + if err != nil { + log.Error("project id conversion failed ", err) + c.JSON(400, gin.H{ + "message": "project id conversion", + }) + return + } + + if err := setCommitStatus(projectIDInt, lastCommitSha, email, string(gitlab.Success)); err != nil { + log.Error("setting commit status failed", err) + c.JSON(500, gin.H{ + "message": "setting commit status failed", + }) + return + } + + log.Infof("email to sign is : %s", email) + log.Infof("project id : %s, sha : %s", projectID, lastCommitSha) + + c.JSON(http.StatusOK, gin.H{ + "message": fmt.Sprintf("user : %s, signed for project : %s", email, projectID), + }) + + }) + + r.POST("/gitlab/events", func(c *gin.Context) { + jsonData, err := ioutil.ReadAll(c.Request.Body) + event, err := gitlab.ParseWebhook(gitlab.EventTypeMergeRequest, jsonData) + if err != nil { + log.Error("parsing json body failed", err) + c.JSON(400, gin.H{ + "message": "code is required parameter", + }) + return + } + + mergeEvent, ok := event.(*gitlab.MergeEvent) + if !ok { + c.JSON(400, gin.H{ + "message": "type cast failed", + }) + return + } + + if mergeEvent.ObjectAttributes.State != "opened" { + c.JSON(200, gin.H{ + "message": "only interested in opened events", + }) + return + } + + projectName := mergeEvent.Project.Name + projectID := mergeEvent.Project.ID + + mergeID := mergeEvent.ObjectAttributes.IID + lastCommitSha := mergeEvent.ObjectAttributes.LastCommit.ID + lastCommitMessage := mergeEvent.ObjectAttributes.LastCommit.Message + + authorName := mergeEvent.ObjectAttributes.LastCommit.Author.Name + authorEmail := mergeEvent.ObjectAttributes.LastCommit.Author.Email + + log.Printf("Received MR (%d) for Project %s:%d", mergeID, projectName, projectID) + log.Printf("last commit : %s : %s", lastCommitSha, lastCommitMessage) + log.Printf("author name : %s, author email : %s", authorName, authorEmail) + + if err := setCommitStatus(projectID, lastCommitSha, authorEmail, ""); err != nil { + log.Error("setting commit status failed", err) + c.JSON(500, gin.H{ + "message": "setting commit status failed", + }) + return + } + + //empJSON, err := json.MarshalIndent(mergeEvent, "", " ") + //if err != nil { + // log.Fatalf(err.Error()) + //} + //fmt.Printf("MarshalIndent funnction output %s\n", string(empJSON)) + c.JSON(http.StatusOK, gin.H{}) + + }) + r.GET("/gitlab/oauth/callback", func(c *gin.Context) { + code := c.Query("code") + if code == "" { + c.JSON(400, gin.H{ + "message": "code is required parameter", + }) + return + } + + state := c.Query("state") + if state == "" { + c.JSON(400, gin.H{ + "message": "state is required parameter", + }) + return + } + log.Printf("received code : %s, STATE: %s", code, state) + + client := resty.New() + params := map[string]string{ + "client_id": clientAppID, + "client_secret": clientAppSecret, + "code": code, + "grant_type": "authorization_code", + "redirect_uri": clientRedirectURI, + } + + resp, err := client.R(). + SetQueryParams(params). + SetResult(&OauthSuccessResponse{}). + Post("https://gitlab.com/oauth/token") + + if err != nil { + c.JSON(500, gin.H{ + "message": fmt.Sprintf("getting the token failed : %v", err), + }) + return + } + + result := resp.Result().(*OauthSuccessResponse) + accessToken := result.AccessToken + + err = registerWebHooksForUserProjects(accessToken) + if err != nil { + log.Error("register webhook ", err) + } + + respData := gin.H{ + "message": "OK", + "data": result, + } + + if err != nil { + respData["error"] = err + } + + c.JSON(200, respData) + }) + r.Run() // listen and serve on 0.0.0.0:8080 (for windows "localhost:8080") +} + +func registerWebHooksForUserProjects(accessToken string) error { + gitlabClient, err := gitlab.NewOAuthClient(accessToken) + if err != nil { + return fmt.Errorf("creating client failed : %v", err) + } + + user, _, err := gitlabClient.Users.CurrentUser() + if err != nil { + return fmt.Errorf("fetching current user failed : %v", err) + } + + log.Infof("fetched current user : %s", user.Name) + + projects, _, err := gitlabClient.Projects.ListUserProjects(user.ID, &gitlab.ListProjectsOptions{}) + if err != nil { + return fmt.Errorf("listing projects failed : %v", err) + } + + log.Printf("we fetched : %d projects for the account", len(projects)) + + for _, p := range projects { + log.Println("**********************") + log.Println("Name : ", p.Name) + log.Println("ID: ", p.ID) + log.Infof("adding webhook to the project : %s (%d)", p.Name, p.ID) + if err := addCLAHookToProject(gitlabClient, p.ID); err != nil { + return fmt.Errorf("adding hook to the project : %s (%d) failed : %v", p.Name, p.ID, err) + } + } + + return nil +} + +func addCLAHookToProject(gitlabClient *gitlab.Client, projectID int) error { + _, _, err := gitlabClient.Projects.AddProjectHook(projectID, &gitlab.AddProjectHookOptions{ + URL: gitlab.String(hookURL), + MergeRequestsEvents: gitlab.Bool(true), + EnableSSLVerification: gitlab.Bool(false), + }) + return err +} + +func setCommitStatus(projectID interface{}, commitSha string, userEmail string, forceState string) error { + accessToken := os.Getenv("GITLAB_ACCESS_TOKEN") + if accessToken == "" { + return fmt.Errorf("GITLAB_ACCESS_TOKEN is required") + } + + gitlabClient, err := gitlab.NewOAuthClient(accessToken) + if err != nil { + return fmt.Errorf("creating client failed : %v", err) + } + + setState := gitlab.Failed + + if forceState == "" { + passingUsersMutex.RLock() + isPassingUser := passingUsers[userEmail] + passingUsersMutex.RUnlock() + + if isPassingUser { + setState = gitlab.Success + } + } else { + setState = gitlab.BuildStateValue(forceState) + } + + options := &gitlab.SetCommitStatusOptions{ + State: setState, + Name: gitlab.String("easyCLA Bot"), + Description: gitlab.String(getDescription(setState)), + } + + if setState == gitlab.Failed { + options.TargetURL = gitlab.String(getTargetURL(projectID, commitSha, userEmail)) + } + + _, _, err = gitlabClient.Commits.SetCommitStatus(projectID, commitSha, options) + if err != nil { + return fmt.Errorf("setting commit status for the sha failed : %v", err) + } + + return nil +} + +func getDescription(status gitlab.BuildStateValue) string { + if status == gitlab.Failed { + return "User hasn't signed CLA" + } + return "User signed CLA" +} + +func getTargetURL(projectID interface{}, lastCommitSha, email string) string { + base := "http://localhost:8080/gitlab/sign" + + projectIDInt := projectID.(int) + projectIDStr := strconv.Itoa(projectIDInt) + + params := url.Values{} + params.Add("project_id", projectIDStr) + params.Add("sha", lastCommitSha) + params.Add("email", email) + + return base + "?" + params.Encode() +} diff --git a/cla-backend-go/cmd/gitlab/branch_protection/main.go b/cla-backend-go/cmd/gitlab/branch_protection/main.go new file mode 100644 index 000000000..244fbd229 --- /dev/null +++ b/cla-backend-go/cmd/gitlab/branch_protection/main.go @@ -0,0 +1,146 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "flag" + "fmt" + "os" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/xanzy/go-gitlab" +) + +const ( + possibleDefaultBranch = "main" +) + +var projectID = flag.Int("project", 0, "gitlab project id") + +func main() { + flag.Parse() + + if *projectID == 0 { + log.Fatalf("gitlab project id is missing") + } + + accessToken := os.Getenv("GITLAB_ACCESS_TOKEN") + if accessToken == "" { + log.Fatalf("GITLAB_ACCESS_TOKEN is required") + } + + gitlabClient, err := gitlab.NewOAuthClient(accessToken) + if err != nil { + log.Fatalf("creating client failed : %v", err) + } + + defaultBranch, err := getDefaultBranch(gitlabClient, *projectID) + if err != nil { + log.Fatalf("fetching the default branch failed : %v", err) + } + + log.Println("the default branch found is : ", defaultBranch.Name) + if err := setOrCreateProtection(gitlabClient, *projectID, defaultBranch.Name); err != nil { + log.Fatalf("setting branch protection for : %s failed : %v", defaultBranch.Name, err) + } + + log.Println("branch protection set for : ", defaultBranch.Name) +} + +func setOrCreateProtection(client *gitlab.Client, projectID int, protectionPattern string) error { + var err error + + protectedBranch, resp, err := client.ProtectedBranches.GetProtectedBranch(projectID, protectionPattern) + if err != nil && resp.StatusCode != 404 { + return fmt.Errorf("fetching existing branch failed : %v", err) + } + + if protectedBranch != nil { + if isProtectedBranchSet(protectedBranch) { + log.Println("branch protection already set, nothing to do") + return nil + } + //it's an existing one try to remove it first and re-create it + log.Println("removing old branch protection for string : ", protectionPattern) + _, err = client.ProtectedBranches.UnprotectRepositoryBranches(projectID, protectionPattern) + if err != nil { + return fmt.Errorf("removing protection for existing branch failed : %v", err) + } + } + + log.Println("re-creating branch protection for string ", protectionPattern) + if _, err = createBranchProtection(client, projectID, protectionPattern); err != nil { + return fmt.Errorf("recreating : %v", err) + } + return nil +} + +func createBranchProtection(client *gitlab.Client, projectID int, name string) (*gitlab.ProtectedBranch, error) { + protectedBranch, _, err := client.ProtectedBranches.ProtectRepositoryBranches(projectID, &gitlab.ProtectRepositoryBranchesOptions{ + Name: gitlab.String(name), + PushAccessLevel: gitlab.AccessLevel(gitlab.NoPermissions), + MergeAccessLevel: gitlab.AccessLevel(gitlab.MaintainerPermissions), + UnprotectAccessLevel: nil, + AllowForcePush: gitlab.Bool(false), + AllowedToPush: nil, + AllowedToMerge: nil, + AllowedToUnprotect: nil, + CodeOwnerApprovalRequired: nil, + }) + if err != nil { + return nil, fmt.Errorf("creating new branch protection failed : %v", err) + } + return protectedBranch, nil +} + +func isProtectedBranchSet(protectedBranch *gitlab.ProtectedBranch) bool { + //log.Println("checking branch protection for : ", spew.Sdump(protectedBranch)) + if protectedBranch.AllowForcePush { + return false + } + + if len(protectedBranch.PushAccessLevels) != 1 { + return false + } + + if protectedBranch.PushAccessLevels[0].AccessLevel != gitlab.NoPermissions { + return false + } + + if len(protectedBranch.MergeAccessLevels) != 1 { + return false + } + + if protectedBranch.MergeAccessLevels[0].AccessLevel != gitlab.MaintainerPermissions { + return false + } + + if len(protectedBranch.UnprotectAccessLevels) != 1 { + return false + } + + if protectedBranch.UnprotectAccessLevels[0].AccessLevel != gitlab.MaintainerPermissions { + return false + } + + return true +} + +// finds the default branch for the given project +func getDefaultBranch(client *gitlab.Client, projectID int) (*gitlab.Branch, error) { + project, _, err := client.Projects.GetProject(projectID, &gitlab.GetProjectOptions{}) + if err != nil { + return nil, fmt.Errorf("fetching project failed : %v", err) + } + + defaultBranch := project.DefaultBranch + + // first try with the possible option + branch, _, err := client.Branches.GetBranch(projectID, defaultBranch) + if err != nil { + return nil, fmt.Errorf("fetching default branch failed : %v", err) + } + + return branch, nil +} diff --git a/cla-backend-go/cmd/gitlab/gitlaborgevents/main.go b/cla-backend-go/cmd/gitlab/gitlaborgevents/main.go new file mode 100644 index 000000000..af880e4ae --- /dev/null +++ b/cla-backend-go/cmd/gitlab/gitlaborgevents/main.go @@ -0,0 +1,69 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "encoding/json" + "os" + + "github.com/aws/aws-lambda-go/events" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" +) + +// This function is useful for generating dynamodb events for GitlabOrg testing, the output of this function is used as +// ./bin/dynamo-events-lambda-mac {OUTPUT_OF_THIS_FUNCTION} +func main() { + // authInfo needed for creating the gitlab client + authInfo := os.Getenv("GITLAB_AUTH_INFO") + + event := events.DynamoDBEvent{ + Records: []events.DynamoDBEventRecord{ + { + EventSourceArn: "aws:dynamodb/cla-dev-gitlab-orgs", + EventName: "MODIFY", + EventSource: "aws:dynamodb", + Change: events.DynamoDBStreamRecord{ + OldImage: map[string]events.DynamoDBAttributeValue{ + "organization_id": events.NewStringAttribute("4ace6f9f-0518-4621-ae86-d0dacc75af83"), + "organization_full_path": events.NewStringAttribute("penguinsoft"), + "organization_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "auto_enabled_cla_group_id": events.NewStringAttribute("40af3652-e8bf-489d-a917-cb2214a89640"), + "external_gitlab_group_id": events.NewNumberAttribute("12700028"), + "auth_state": events.NewStringAttribute("18eb90d1-8c36-4962-ba91-e264ccbcab3a"), + "organization_url": events.NewStringAttribute("https://gitlab.com/groups/penguinsoft"), + "auth_info": events.NewStringAttribute(authInfo), + "organization_name_lower": events.NewStringAttribute("penguinsoft"), + "project_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "organization_name": events.NewStringAttribute("penguinsoft"), + "enabled": events.NewBooleanAttribute(false), + "auto_enabled": events.NewBooleanAttribute(false), + "branch_protection_enabled": events.NewBooleanAttribute(false), + }, + NewImage: map[string]events.DynamoDBAttributeValue{ + "organization_id": events.NewStringAttribute("4ace6f9f-0518-4621-ae86-d0dacc75af83"), + "organization_full_path": events.NewStringAttribute("penguinsoft"), + "organization_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "auto_enabled_cla_group_id": events.NewStringAttribute("40af3652-e8bf-489d-a917-cb2214a89640"), + "external_gitlab_group_id": events.NewNumberAttribute("12700028"), + "auth_state": events.NewStringAttribute("18eb90d1-8c36-4962-ba91-e264ccbcab3a"), + "organization_url": events.NewStringAttribute("https://gitlab.com/groups/penguinsoft"), + "auth_info": events.NewStringAttribute(authInfo), + "organization_name_lower": events.NewStringAttribute("penguinsoft"), + "project_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "organization_name": events.NewStringAttribute("penguinsoft"), + "enabled": events.NewBooleanAttribute(true), + "auto_enabled": events.NewBooleanAttribute(false), + "branch_protection_enabled": events.NewBooleanAttribute(true), + }, + }}, + }, + } + + b, err := json.Marshal(event) + if err != nil { + log.Fatalf("marshall : %v", err) + } + + log.Println(string(b)) +} diff --git a/cla-backend-go/cmd/gitlab/project_settings/main.go b/cla-backend-go/cmd/gitlab/project_settings/main.go new file mode 100644 index 000000000..7773ade15 --- /dev/null +++ b/cla-backend-go/cmd/gitlab/project_settings/main.go @@ -0,0 +1,40 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "flag" + "os" + + gitlab_api "github.com/linuxfoundation/easycla/cla-backend-go/gitlab_api" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/xanzy/go-gitlab" +) + +var projectID = flag.Int("project", 0, "gitlab project id") + +func main() { + flag.Parse() + + if *projectID == 0 { + log.Fatalf("gitlab project id is missing") + } + + accessToken := os.Getenv("GITLAB_ACCESS_TOKEN") + if accessToken == "" { + log.Fatalf("GITLAB_ACCESS_TOKEN is required") + } + + gitlabClient, err := gitlab.NewOAuthClient(accessToken) + if err != nil { + log.Fatalf("creating client failed : %v", err) + } + + if err := gitlab_api.EnableMergePipelineProtection(context.Background(), gitlabClient, *projectID); err != nil { + log.Fatalf("enabling merge pipeline protection failed : %v", err) + } + + log.Println("merge pipeline protection enabled successfully") +} diff --git a/cla-backend-go/cmd/gitlab/repoevents/main.go b/cla-backend-go/cmd/gitlab/repoevents/main.go new file mode 100644 index 000000000..010ec74dd --- /dev/null +++ b/cla-backend-go/cmd/gitlab/repoevents/main.go @@ -0,0 +1,55 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "encoding/json" + + "github.com/aws/aws-lambda-go/events" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" +) + +// This function is useful for generating dynamodb events for Repository testing, the output of this function is used as +// ./bin/dynamo-events-lambda-mac {OUTPUT_OF_THIS_FUNCTION} +func main() { + event := events.DynamoDBEvent{ + Records: []events.DynamoDBEventRecord{ + { + EventSourceArn: "aws:dynamodb/cla-dev-repositories", + EventName: "MODIFY", + EventSource: "aws:dynamodb", + Change: events.DynamoDBStreamRecord{ + OldImage: map[string]events.DynamoDBAttributeValue{ + "repository_id": events.NewStringAttribute("1fa3de39-8274-4750-ba7c-242d5d659dd1"), + "repository_name": events.NewStringAttribute("easycla-gitlab-test"), + "repository_organization_name": events.NewStringAttribute("penguinsoft"), + "repository_project_id": events.NewStringAttribute("40af3652-e8bf-489d-a917-cb2214a89640"), + "repository_sfdc_id": events.NewStringAttribute("a092M00001If9uZQAR"), + "project_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "repository_external_id": events.NewNumberAttribute("28893091"), + "repository_type": events.NewStringAttribute("gitlab"), + "enabled": events.NewBooleanAttribute(false), + }, + NewImage: map[string]events.DynamoDBAttributeValue{ + "repository_id": events.NewStringAttribute("1fa3de39-8274-4750-ba7c-242d5d659dd1"), + "repository_name": events.NewStringAttribute("easycla-gitlab-test"), + "repository_organization_name": events.NewStringAttribute("penguinsoft"), + "repository_project_id": events.NewStringAttribute("40af3652-e8bf-489d-a917-cb2214a89640"), + "repository_sfdc_id": events.NewStringAttribute("a092M00001If9uZQAR"), + "project_sfid": events.NewStringAttribute("a092M00001If9uZQAR"), + "repository_external_id": events.NewNumberAttribute("28893091"), + "repository_type": events.NewStringAttribute("gitlab"), + "enabled": events.NewBooleanAttribute(true), + }, + }}, + }, + } + + b, err := json.Marshal(event) + if err != nil { + log.Fatalf("marshall : %v", err) + } + + log.Println(string(b)) +} diff --git a/cla-backend-go/cmd/gitlab/webhook/main.go b/cla-backend-go/cmd/gitlab/webhook/main.go new file mode 100644 index 000000000..0cb5ebb7f --- /dev/null +++ b/cla-backend-go/cmd/gitlab/webhook/main.go @@ -0,0 +1,88 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "os" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/xanzy/go-gitlab" +) + +const ( + hookURL = "https://7e182f2774e2.ngrok.io/gitlab/events" +) + +func main() { + log.Println("register webhook") + access_token := os.Getenv("GITLAB_ACCESS_TOKEN") + if access_token == "" { + log.Fatal("GITLAB_ACCESS_TOKEN is required") + } + + log.Infof("The gitlab access token is : %s", access_token) + + gitlabClient, err := gitlab.NewOAuthClient(access_token) + if err != nil { + log.Fatalf("creating client failed : %v", err) + } + + user, _, err := gitlabClient.Users.CurrentUser() + if err != nil { + log.Fatalf("fetching current user failed : %v", err) + } + + log.Infof("fetched current user : %s", user.Name) + + projects, _, err := gitlabClient.Projects.ListUserProjects(user.ID, &gitlab.ListProjectsOptions{}) + if err != nil { + log.Fatalf("listing projects failed : %v", err) + } + log.Printf("we fetched : %d projects for the account", len(projects)) + for _, p := range projects { + log.Println("**********************") + log.Println("Name : ", p.Name) + log.Println("ID: ", p.ID) + hooks, _, err := gitlabClient.Projects.ListProjectHooks(p.ID, &gitlab.ListProjectHooksOptions{}) + + if err != nil { + log.Fatalf("fetching hooks for project : %s, failed : %v", p.Name, err) + } + + var claHookFound bool + for _, hook := range hooks { + log.Println("**********************") + log.Infof("hook ID : %d", hook.ID) + log.Infof("URL : %s", hook.URL) + log.Infof("Merge Request Events Enabled : %v", hook.MergeRequestsEvents) + log.Infof("Enable SSL Verification : %v", hook.EnableSSLVerification) + + if hookURL == hook.URL { + claHookFound = true + break + } + } + + if claHookFound { + log.Infof("CLA Hook was found nothing to do") + continue + } + + log.Infof("adding webhook to the project : %s (%d)", p.Name, p.ID) + if err := addCLAHookToProject(gitlabClient, p.ID); err != nil { + log.Fatalf("adding hook to the project : %s (%d) failed : %v", p.Name, p.ID, err) + } + + } + +} + +func addCLAHookToProject(gitlabClient *gitlab.Client, projectID int) error { + _, _, err := gitlabClient.Projects.AddProjectHook(projectID, &gitlab.AddProjectHookOptions{ + URL: gitlab.String(hookURL), + MergeRequestsEvents: gitlab.Bool(true), + EnableSSLVerification: gitlab.Bool(false), + }) + return err +} diff --git a/cla-backend-go/cmd/gitlab_repository_check/README.md b/cla-backend-go/cmd/gitlab_repository_check/README.md new file mode 100644 index 000000000..0133cc198 --- /dev/null +++ b/cla-backend-go/cmd/gitlab_repository_check/README.md @@ -0,0 +1,22 @@ +# GitLab Repository Check Lambda + +GitLab (currently) does not support sending callback/webhook events for GitLab project add or delete events. As a +result, we created a small lambda that runs periodically to check for any new GitLab project +(repository) add or deletes. + +The process/algorithm is: + +1. Query our database for registered GitLab Groups - filter by the enabled flag is true and where the Auto Enable flag + is true +1. For each GitLab group in our database... + 1. Create a new GitLab API client instance using the authorization token for the Git Group + 1. Query the GitLab API for the project list under the group (include sub-groups). This grabs the list of current + GitLab projects under the GitLab group. + 1. Query for GitLab project in DB matching this GitLab group path + 1. Identify deltas - this identifies how many new and deleted GitLap projects we need to process + 1. If any new GitLab projects, add to the DB, set enabled, create an event log + 1. If any removed/deleted GitLab projects, remove from the DB, create an event log + +## References + +- [GitLab Feature request discussion thread](https://gitlab.com/gitlab-com/marketing/community-relations/opensource-program/linux-foundation/-/issues/4#note_653255564) diff --git a/cla-backend-go/cmd/gitlab_repository_check/handler/handler.go b/cla-backend-go/cmd/gitlab_repository_check/handler/handler.go new file mode 100644 index 000000000..4bee92b7a --- /dev/null +++ b/cla-backend-go/cmd/gitlab_repository_check/handler/handler.go @@ -0,0 +1,367 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package handler + +import ( + "context" + "os" + "strconv" + + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/common" + + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + + "github.com/linuxfoundation/easycla/cla-backend-go/config" + + "github.com/aws/aws-sdk-go/aws/session" + + v1Company "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + gitLabApi "github.com/linuxfoundation/easycla/cla-backend-go/gitlab_api" + gitlab "github.com/linuxfoundation/easycla/cla-backend-go/gitlab_api" + ini "github.com/linuxfoundation/easycla/cla-backend-go/init" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + v1Repositories "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/gitlab_organizations" + v2Repositories "github.com/linuxfoundation/easycla/cla-backend-go/v2/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/store" + + "strings" + + "github.com/sirupsen/logrus" + goGitLab "github.com/xanzy/go-gitlab" +) + +var ( + awsSession *session.Session + stage string + configFile config.Config + gitLabApp *gitLabApi.App +) + +// Init initializes the handler +func Init() { + f := logrus.Fields{ + "functionName": "cmd.gitlab_repository_check.handler.Init", + } + ctx := utils.NewContext() + f[utils.XREQUESTID] = ctx.Value(utils.XREQUESTID) + log.WithFields(f).Debug("initializing...") + + // General initialization + ini.Init() + + var awsErr error + awsSession, awsErr = ini.GetAWSSession() + if awsErr != nil { + log.WithFields(f).WithError(awsErr).Panic("unable to load AWS session") + } + + // Need to initialize the system to load the configuration which contains a number of SSM parameters + stage = os.Getenv("STAGE") + if stage == "" { + log.WithFields(f).Panic("unable to determine STAGE - please set in the environment variable: 'STAGE' - expected one of [DEV, STAGING, PROD]") + } + + dynamodbRegion := os.Getenv("DYNAMODB_AWS_REGION") + if dynamodbRegion == "" { + log.WithFields(f).Panic("unable to determine DYNAMODB_AWS_REGION - please set in the environment variable: 'DYNAMODB_AWS_REGION'") + } + + var configErr error + configFile, configErr = config.LoadConfig("", awsSession, stage) + if configErr != nil { + log.WithFields(f).WithError(configErr).Panicf("Unable to load config - Error: %v", configErr) + } + + if configFile.Gitlab.AppClientID == "" { + log.WithFields(f).Panic("unable to determine configFile.Gitlab.AppClientID value - please set the configuration") + } + if configFile.Gitlab.AppClientSecret == "" { + log.WithFields(f).Panic("unable to determine configFile.Gitlab.AppClientSecret value - please set the configuration") + } + if configFile.Gitlab.AppPrivateKey == "" { + log.WithFields(f).Panic("unable to determine configFile.Gitlab.AppPrivateKey value - please set the configuration") + } + + // Create a new GitLab App client instance + gitLabApp = gitlab.Init(configFile.Gitlab.AppClientID, configFile.Gitlab.AppClientSecret, configFile.Gitlab.AppPrivateKey) + +} + +// Handler is invoked each time the lambda is triggered - https://docs.aws.amazon.com/lambda/latest/dg/golang-handler.html +func Handler(ctx context.Context) error { + f := logrus.Fields{ + "functionName": "cmd.update-project-statistics.Handler", + } + + // Add the x-request-id to the context + ctx = utils.NewContextFromParent(ctx) + f[utils.XREQUESTID] = ctx.Value(utils.XREQUESTID) + + // Repository Layer + usersRepo := users.NewRepository(awsSession, stage) + eventsRepo := events.NewRepository(awsSession, stage) + v1CompanyRepo := v1Company.NewRepository(awsSession, stage) + gerritRepo := gerrits.NewRepository(awsSession, stage) + v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) + gitV1Repository := v1Repositories.NewRepository(awsSession, stage) + gitV2Repository := v2Repositories.NewRepository(awsSession, stage) + githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) + gitlabOrganizationRepo := gitlab_organizations.NewRepository(awsSession, stage) + v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo) + storeRepo := store.NewRepository(awsSession, stage) + + // Service Layer + + type combinedRepo struct { + users.UserRepository + v1Company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository + } + + // Our service layer handlers + eventsService := events.NewService(eventsRepo, combinedRepo{ + usersRepo, + v1CompanyRepo, + v1CLAGroupRepo, + v1ProjectClaGroupRepo, + }) + + gerritService := gerrits.NewService(gerritRepo) + + approvalsTableName := "cla-" + stage + "-approvals" + + usersService := users.NewService(usersRepo, eventsService) + approvalsRepo := approvals.NewRepository(stage, awsSession, approvalsTableName) + signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService, gitV1Repository, githubOrganizationsRepo, gerritService, approvalsRepo) + v2RepositoriesService := v2Repositories.NewService(gitV1Repository, gitV2Repository, v1ProjectClaGroupRepo, githubOrganizationsRepo, gitlabOrganizationRepo, eventsService) + // gitlabOrganizationsService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoriesService, v1ProjectClaGroupRepo) + gitlabOrganizationService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoriesService, v1ProjectClaGroupRepo, storeRepo, usersService, signaturesRepo, v1CompanyRepo) + + // Query GitLab Groups + // for each group + // if enabled and auto-enabled = true + // load token and client + // query for GitLab API repository list + // query for GitLab repositories in DB for this group path + // identify deltas + // if new, add to DB, create event log + // if deleted, remove from DB, create event log + + // gitLabGroups, err := gitlabOrganizationRepo.GetGitLabOrganizationsEnabled(ctx) + // if err != nil { + // log.WithFields(f).WithError(err).Warnf("problem querying for GitLab group/organizations that are enabled with auto-enabled flag set to true") + // return err + // } + gitLabGroups, err := gitlabOrganizationRepo.GetGitLabOrganizationsByProjectSFID(ctx, "a092h000004x5sGAAQ") + if err != nil { + log.WithFields(f).WithError(err).Warnf("problem querying for GitLab group/organizations that are enabled with auto-enabled flag set to true") + return err + } + + log.WithFields(f).Debugf("start - checking %d GitLab projects for add/delete events", len(gitLabGroups.List)) + for _, gitLabGroup := range gitLabGroups.List { + claGroupID := gitLabGroup.AutoEnabledClaGroupID + log.WithFields(f).Debugf("start - processing GitLab group/organization: %s with group ID: %d associated with project SFID: %s", gitLabGroup.OrganizationURL, gitLabGroup.OrganizationExternalID, gitLabGroup.ProjectSfid) + + if claGroupID == "" { + log.WithFields(f).Debugf("GitLab group/organization: %s not fully onboarded - missing CLA Group ID", gitLabGroup.OrganizationURL) + pcg, err := v1ProjectClaGroupRepo.GetClaGroupIDForProject(ctx, gitLabGroup.ProjectSfid) + if err != nil { + log.WithFields(f).WithError(err).Warnf("problem querying for CLA Group ID for project SFID: %s", gitLabGroup.ProjectSfid) + continue + } + log.WithFields(f).Debug("found CLA Group ID: ", pcg.ClaGroupID) + claGroupID = pcg.ClaGroupID + } + + if gitLabGroup.AuthInfo == "" { + log.WithFields(f).Debugf("GitLab group/organization: %s not fully onboarded - missing authentication info - skipping", gitLabGroup.OrganizationURL) + continue + } + + oauthResponse, err := gitlabOrganizationService.RefreshGitLabOrganizationAuth(ctx, common.ToCommonModel(gitLabGroup)) + + if err != nil { + log.WithFields(f).WithError(err).Warnf("problem refreshing GitLab group/organization: %s authentication info - skipping", gitLabGroup.OrganizationURL) + continue + } + + gitLabClient, gitLabClientErr := gitLabApi.NewGitlabOauthClient(*oauthResponse, gitLabApp) + if gitLabClientErr != nil { + log.WithFields(f).WithError(gitLabClientErr).Warnf("problem loading GitLab client for group/organization: %s - skipping", gitLabGroup.OrganizationURL) + continue + } + + gitLabProjects, getGitLabAPIError := gitLabApi.GetGroupProjectListByGroupID(ctx, gitLabClient, int(gitLabGroup.OrganizationExternalID)) + if getGitLabAPIError != nil { + log.WithFields(f).WithError(getGitLabAPIError).Warnf("problem loading GitLab projects for group/organization: %s using the groupID: %d - skipping GitLab Group/Organziation - skipping", gitLabGroup.OrganizationFullPath, gitLabGroup.OrganizationExternalID) + continue + } + log.WithFields(f).Debugf("found %d GitLab projects for group/organization: %s", len(gitLabProjects), gitLabGroup.OrganizationFullPath) + + gitLabDBProjects, getProjectListDBErr := gitV2Repository.GitLabGetRepositoriesByOrganizationName(ctx, gitLabGroup.OrganizationFullPath) + if getProjectListDBErr != nil { + if _, ok := getProjectListDBErr.(*utils.GitLabRepositoryNotFound); ok { + log.WithFields(f).Debugf("GitLab group/organization: %s does not have any repositories in the database", gitLabGroup.OrganizationFullPath) + } else { + log.WithFields(f).WithError(getProjectListDBErr).Warnf("problem loading GitLab projects for group/organization: %s from the database - skipping GitLab Group/Organziation - skipping", gitLabGroup.OrganizationFullPath) + continue + } + } + + newGitLabProjects := getNewProjects(gitLabProjects, gitLabDBProjects) + log.WithFields(f).Debugf("Found %d GitLab projects/repositories that are to be added for GitLab Group: %s", len(newGitLabProjects), gitLabGroup.OrganizationFullPath) + if len(newGitLabProjects) > 0 { + var gitLabProjectIDList []int64 + + // Build a quick list of the GitLab Project/repo ID values - the add repositories takes a list + for _, newGitLabProject := range newGitLabProjects { + gitLabProjectIDList = append(gitLabProjectIDList, int64(newGitLabProject.ID)) + } + + // Add the repositories - will generate a log event + _, addErr := v2RepositoriesService.GitLabAddRepositoriesWithEnabledFlag(ctx, gitLabGroup.ProjectSfid, &v2Repositories.GitLabAddRepoModel{ + ClaGroupID: claGroupID, + GroupName: gitLabGroup.OrganizationName, + ExternalID: gitLabGroup.OrganizationExternalID, + GroupFullPath: gitLabGroup.OrganizationFullPath, + ProjectIDList: gitLabProjectIDList, + }, true) // set to enabled when adding since this was added as a result of the auto-enable feature + if addErr != nil { + log.WithFields(f).WithError(addErr).Warnf("problem adding GitLab projects for group/organization: %s to the database", gitLabGroup.OrganizationFullPath) + } else { + log.WithFields(f).Debugf("added %d GitLab projects for group/organization: %s to the database", len(newGitLabProjects), gitLabGroup.OrganizationFullPath) + } + } + + gitLabProjects, getGitLabAPIError = gitLabApi.GetGroupProjectListByGroupID(ctx, gitLabClient, int(gitLabGroup.OrganizationExternalID)) + if getGitLabAPIError != nil { + log.WithFields(f).WithError(getGitLabAPIError).Warnf("problem loading GitLab projects for group/organization: %s using the groupID: %d - skipping GitLab Group/Organziation - skipping", gitLabGroup.OrganizationFullPath, gitLabGroup.OrganizationExternalID) + continue + } + log.WithFields(f).Debugf("found %d GitLab projects for group/organization: %s", len(gitLabProjects), gitLabGroup.OrganizationFullPath) + + dBProjects, getProjectListDBErr := gitV2Repository.GitLabGetRepositoriesByOrganizationName(ctx, gitLabGroup.OrganizationFullPath) + if getProjectListDBErr != nil { + if _, ok := getProjectListDBErr.(*utils.GitLabRepositoryNotFound); ok { + log.WithFields(f).Debugf("GitLab group/organization: %s does not have any repositories in the database", gitLabGroup.OrganizationFullPath) + } else { + log.WithFields(f).WithError(getProjectListDBErr).Warnf("problem loading GitLab projects for group/organization: %s from the database - skipping GitLab Group/Organziation - skipping", gitLabGroup.OrganizationFullPath) + continue + } + } + log.WithFields(f).Debugf("Found %d GitLab projects/repositories for GitLab Group: %s", len(dBProjects), gitLabGroup.OrganizationFullPath) + + deletedGitLabProjects := getDeletedProjects(gitLabProjects, dBProjects) + log.WithFields(f).Debugf("Found %d GitLab projects/repositories that are to be removed from the GitLab Group: %s", len(deletedGitLabProjects), gitLabGroup.OrganizationFullPath) + if len(deletedGitLabProjects) > 0 { + for _, gitLabProjectDBRecord := range deletedGitLabProjects { + repositoryExternalID, parseIntErr := strconv.ParseInt(gitLabProjectDBRecord.RepositoryExternalID, 10, 64) + if parseIntErr != nil { + log.WithFields(f).WithError(parseIntErr).Warnf("problem converting repository %s external ID string value: %s to integer", gitLabProjectDBRecord.RepositoryFullPath, gitLabProjectDBRecord.RepositoryExternalID) + } else { + deleteErr := v2RepositoriesService.GitLabDeleteRepositoryByExternalID(ctx, repositoryExternalID) + if deleteErr != nil { + log.WithFields(f).WithError(deleteErr).Warnf("problem deleting repository %s external ID string value: %s to integer", gitLabProjectDBRecord.RepositoryFullPath, gitLabProjectDBRecord.RepositoryExternalID) + } else { + log.WithFields(f).Debugf("deleted GitLab project %s for group/organization: %s from the database", gitLabProjectDBRecord.RepositoryName, gitLabGroup.OrganizationFullPath) + } + } + } + } + + log.WithFields(f).Debugf("done - processed GitLab group/organization: %s with group ID: %d associated with project SFID: %s", gitLabGroup.OrganizationURL, gitLabGroup.OrganizationExternalID, gitLabGroup.ProjectSfid) + } + + log.WithFields(f).Debugf("done - checked %d GitLab projects for add/delete events", len(gitLabGroups.List)) + return nil +} + +// getNewProjects is a helper function to determine if we have any new GitLab projects that are not in our database +func getNewProjects(gitLabProjects []*goGitLab.Project, gitLabDBProjects []*v1Repositories.RepositoryDBModel) []*goGitLab.Project { + var response []*goGitLab.Project + f := logrus.Fields{ + "functionName": "getNewProjects", + } + if len(gitLabDBProjects) == 0 { + // No projects in the database - return all the projects from GitLab + log.WithFields(f).Debugf("no projects in the database - returning all projects from GitLab: %+v", gitLabProjects) + return gitLabProjects + } + + // For each GitLab Project/Repo + for _, gitLabProject := range gitLabProjects { + found := false + + // For each GitLab Project/Repo in the database + for _, gitLabDBProject := range gitLabDBProjects { + // Compare the full name/path + if strings.ToLower(gitLabProject.PathWithNamespace) == strings.ToLower(gitLabDBProject.RepositoryFullPath) { + found = true + break + } + } + + // Didn't find the GitLab Project Repo from GitLab defined in our database - must have been added! + if !found { + // Add to our list + response = append(response, gitLabProject) + } + } + + return response +} + +// getDeletedProjects is a helper function to determine if we have any new GitLab projects that were removed from GitLab but are still in our database +func getDeletedProjects(gitLabProjects []*goGitLab.Project, gitLabDBProjects []*v1Repositories.RepositoryDBModel) []*v1Repositories.RepositoryDBModel { + response := make([]*v1Repositories.RepositoryDBModel, 0) + f := logrus.Fields{ + "functionName": "getDeletedProjects", + } + + if len(gitLabProjects) == 0 { + // No projects in GitLab - return all the projects from the database + log.WithFields(f).Debugf("no projects in GitLab - returning all projects from the database: %+v", gitLabDBProjects) + return gitLabDBProjects + } + + log.WithFields(f).Debugf("len(gitLabProjects): %d and len(gitLabDbProjects): %d", len(gitLabProjects), len(gitLabDBProjects)) + + // For each GitLab Project/Repo in the database + for _, gitLabDBProject := range gitLabDBProjects { + found := false + + // For each GitLab Project/Repo + for _, gitLabProject := range gitLabProjects { + // Compare the full name/path + log.WithFields(f).Debugf("comparing GitLab project: %s with GitLab DB project: %s", gitLabProject.PathWithNamespace, gitLabDBProject.RepositoryFullPath) + if strings.ToLower(gitLabProject.PathWithNamespace) == strings.ToLower(gitLabDBProject.RepositoryFullPath) { + found = true + break + } + + } + + // Didn't find the GitLab Project Repo from the database defined in GitLab - must have been removed! + if !found { + // Add to our list + log.WithFields(f).Debugf("adding GitLab project: %s to the list of projects to be deleted", gitLabDBProject.RepositoryFullPath) + response = append(response, gitLabDBProject) + } else { + log.WithFields(f).Debugf("GitLab project: %s was not found in the list of projects to be deleted", gitLabDBProject.RepositoryFullPath) + } + } + + return response +} diff --git a/cla-backend-go/cmd/gitlab_repository_check/handler/server_aws_lambda.go b/cla-backend-go/cmd/gitlab_repository_check/handler/server_aws_lambda.go new file mode 100644 index 000000000..144a6980b --- /dev/null +++ b/cla-backend-go/cmd/gitlab_repository_check/handler/server_aws_lambda.go @@ -0,0 +1,23 @@ +//go:build aws_lambda +// +build aws_lambda + +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package handler + +import ( + "github.com/aws/aws-lambda-go/lambda" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/sirupsen/logrus" +) + +// RunHandler starts the lambda main handler routine +func RunHandler() { + f := logrus.Fields{ + "functionName": "cmd.gitlab_repository_check.handler.RunHandler", + } + log.WithFields(f).Info("lambda server starting...") + lambda.Start(Handler) + log.WithFields(f).Infof("Lambda shutting down...") +} diff --git a/cla-backend-go/cmd/gitlab_repository_check/handler/server_standalone.go b/cla-backend-go/cmd/gitlab_repository_check/handler/server_standalone.go new file mode 100644 index 000000000..6d2d068d9 --- /dev/null +++ b/cla-backend-go/cmd/gitlab_repository_check/handler/server_standalone.go @@ -0,0 +1,26 @@ +//go:build !aws_lambda +// +build !aws_lambda + +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package handler + +import ( + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" +) + +// RunHandler starts the lambda in local testing model by invoking the handler directly +func RunHandler() { + f := logrus.Fields{ + "functionName": "cmd.gitlab_repository_check.handler.RunHandler", + } + log.WithFields(f).Debug("creating a new handler") + err := Handler(utils.NewContext()) + if err != nil { + log.WithFields(f).WithError(err).Warn("error returned from handler") + } + log.Infof("handler completed") +} diff --git a/cla-backend-go/cmd/gitlab_repository_check/main.go b/cla-backend-go/cmd/gitlab_repository_check/main.go new file mode 100644 index 000000000..f6480c7c3 --- /dev/null +++ b/cla-backend-go/cmd/gitlab_repository_check/main.go @@ -0,0 +1,11 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import "github.com/linuxfoundation/easycla/cla-backend-go/cmd/gitlab_repository_check/handler" + +func main() { + handler.Init() + handler.RunHandler() +} diff --git a/cla-backend-go/cmd/ldap_gerrit_check/main.go b/cla-backend-go/cmd/ldap_gerrit_check/main.go new file mode 100644 index 000000000..3faa06e7c --- /dev/null +++ b/cla-backend-go/cmd/ldap_gerrit_check/main.go @@ -0,0 +1,173 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + // "context" + "encoding/csv" + "flag" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + eventOps "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/events" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + // "github.com/linuxfoundation/easycla/cla-backend-go/users" +) + +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var stage string + +func main() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + log.Infof("STAGE set to %s\n", stage) + + var wg sync.WaitGroup + var mu sync.Mutex + + // Initialize the events repository + eventsRepo := events.NewRepository(awsSession, stage) + eventService := events.NewService(eventsRepo, nil) + + // Initialize the users repository + // usersRepo := users.NewRepository(awsSession, stage) + + inputFilename := flag.String("input-file", "", "Input with a given list of lf usernames") + claGroup := flag.String("cla-group-id", "", "The ID of the CLA group") + claGroupName := flag.String("cla-group-name", "", "The name of the CLA group") + flag.Parse() + + if *inputFilename == "" || *claGroup == "" { + log.Fatalf("Both input-file and cla-group are required") + } + + log.Debugf("Input file: %s", *inputFilename) + + file, err := os.Open(*inputFilename) + if err != nil { + log.Fatalf("Unable to read input file: %s", *inputFilename) + } + + defer func() { + if err = file.Close(); err != nil { + log.Fatalf("Error closing file: %v", err) + } + }() + + reader := csv.NewReader(file) + + records, err := reader.ReadAll() + if err != nil { + log.Fatalf("Unable to read file") + } + + log.Debugf("CLA Group Name: %s", *claGroup) + + type Report struct { + Username string + Events []*models.Event + } + + projectReport := make([]Report, 0) + + for i, record := range records { + if i == 0 { + continue + } + lfUsername := record[0] + log.Debugf("Processing record: %s", lfUsername) + report := Report{ + Username: lfUsername, + } + + // Increment the wait group + wg.Add(1) + + go func(lfusername string) { + defer wg.Done() + log.Debugf("Processing record: %s", lfusername) + searchParams := eventOps.SearchEventsParams{ + SearchTerm: &lfusername, + ProjectID: claGroup, + } + events, eventErr := eventService.SearchEvents(&searchParams) + if eventErr != nil { + log.Debugf("Error getting events: %v", eventErr) + report.Events = nil + } + + if len(events.Events) == 0 { + log.Warnf("No events found for user: %s", lfusername) + report.Events = nil + } else { + log.Debugf("Events found for user: %s", lfusername) + report.Events = events.Events + } + + mu.Lock() + projectReport = append(projectReport, report) + defer mu.Unlock() + + }(lfUsername) + } + + // Wait for all the go routines to finish + wg.Wait() + + // Create a csv file with the results + outputFilename := fmt.Sprintf("ldap-%s-%s.csv", *claGroupName, time.Now().Format("2006-01-02-15-04-05")) + outputFile, err := os.Create(filepath.Clean(outputFilename)) + + if err != nil { + log.Fatalf("Unable to create output file: %s", outputFilename) + } + + defer func() { + if err = outputFile.Close(); err != nil { + log.Fatalf("Error closing file: %v", err) + } + }() + + writer := csv.NewWriter(outputFile) + + err = writer.Write([]string{"Username", "Event ID", "Event Data", "Event Type", "Event Date"}) + if err != nil { + log.Fatalf("Error writing csv: %v", err) + } + + for _, report := range projectReport { + if report.Events == nil { + err = writer.Write([]string{report.Username, "No events found", "", "", ""}) + if err != nil { + log.Fatalf("Error writing csv: %v", err) + } + continue + } + for _, event := range report.Events { + err = writer.Write([]string{report.Username, event.EventID, event.EventData, event.EventType, event.EventTime}) + if err != nil { + log.Fatalf("Error writing csv: %v", err) + } + + } + } + + writer.Flush() + + if err := writer.Error(); err != nil { + log.Fatalf("Error writing csv: %v", err) + } + + log.Infof("Output written to: %s", outputFilename) + +} diff --git a/cla-backend-go/cmd/metrics_lambda/main.go b/cla-backend-go/cmd/metrics_lambda/main.go index 9c632da1b..001ece6f1 100644 --- a/cla-backend-go/cmd/metrics_lambda/main.go +++ b/cla-backend-go/cmd/metrics_lambda/main.go @@ -7,21 +7,21 @@ import ( "context" "os" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" "github.com/aws/aws-lambda-go/lambda" - "github.com/communitybridge/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/token" - "github.com/communitybridge/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/config" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/v2/metrics" - project_service "github.com/communitybridge/easycla/cla-backend-go/v2/project-service" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/metrics" + project_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service" ) var ( diff --git a/cla-backend-go/cmd/metrics_report_lambda/main.go b/cla-backend-go/cmd/metrics_report_lambda/main.go index b5e752386..2dd09dc1a 100644 --- a/cla-backend-go/cmd/metrics_report_lambda/main.go +++ b/cla-backend-go/cmd/metrics_report_lambda/main.go @@ -13,23 +13,23 @@ import ( "github.com/davecgh/go-spew/spew" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" "github.com/aws/aws-lambda-go/lambda" - "github.com/communitybridge/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/token" - "github.com/communitybridge/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/config" "github.com/LF-Engineering/lfx-models/models/stats" "github.com/aws/aws-lambda-go/events" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" awsqs "github.com/aws/aws-sdk-go/service/sqs" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/v2/metrics" - v2ProjectService "github.com/communitybridge/easycla/cla-backend-go/v2/project-service" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/metrics" + v2ProjectService "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service" ) var ( diff --git a/cla-backend-go/cmd/migrate_approval_list/main.go b/cla-backend-go/cmd/migrate_approval_list/main.go new file mode 100644 index 000000000..b9c423df9 --- /dev/null +++ b/cla-backend-go/cmd/migrate_approval_list/main.go @@ -0,0 +1,303 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "flag" + "fmt" + "os" + "strings" + + // "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/gofrs/uuid" + "github.com/sirupsen/logrus" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + v1Models "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" +) + +var stage string +var approvalRepo approvals.IRepository +var signatureRepo signatures.SignatureRepository +var eventsRepo events.Repository +var usersRepo users.UserRepository +var eventsService events.Service +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var approvalsTableName string +var companyRepo company.IRepository +var v1ProjectClaGroupRepo projects_cla_groups.Repository +var ghRepo repositories.Repository +var gerritsRepo gerrits.Repository +var ghOrgRepo github_organizations.Repository +var gerritService gerrits.Service +var eventsByType []*v1Models.Event +var toUpdateApprovalItems []approvals.ApprovalItem +var toUpdateApprovalItemsMutex sync.Mutex // Protects toUpdateApprovalItems slice + +type combinedRepo struct { + users.UserRepository + company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + + log.Infof("STAGE set to %s\n", stage) + approvalsTableName = fmt.Sprintf("cla-%s-approvals", stage) + approvalRepo = approvals.NewRepository(stage, awsSession, approvalsTableName) + eventsRepo = events.NewRepository(awsSession, stage) + usersRepo = users.NewRepository(awsSession, stage) + companyRepo = company.NewRepository(awsSession, stage) + ghRepo = *repositories.NewRepository(awsSession, stage) + gerritsRepo = gerrits.NewRepository(awsSession, stage) + v1CLAGroupRepo := repository.NewRepository(awsSession, stage, &ghRepo, gerritsRepo, v1ProjectClaGroupRepo) + v1ProjectClaGroupRepo = projects_cla_groups.NewRepository(awsSession, stage) + eventsService = events.NewService(eventsRepo, combinedRepo{ + usersRepo, + companyRepo, + v1CLAGroupRepo, + v1ProjectClaGroupRepo, + }) + ghOrgRepo = github_organizations.NewRepository(awsSession, stage) + gerritService = gerrits.NewService(gerritsRepo) + signatureRepo = signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, eventsService, &ghRepo, ghOrgRepo, gerritService, approvalRepo) + + log.Info("initialized repositories\n") +} + +func main() { + f := logrus.Fields{ + "functionName": "main", + } + log.WithFields(f).Info("Starting migration") + log.Info("Fetching ccla signatures") + signed := true + approved := true + // signatureID := flag.String("signature-id", "ALL", "signature ID to migrate") + delete := flag.Bool("delete", false, "delete approval items") + signatureID := flag.String("signature-id", "ALL", "signature ID to migrate") + flag.Parse() + + if *delete { + log.Info("Deleting approval items") + err := approvalRepo.BatchDeleteApprovalList() + if err != nil { + log.WithFields(f).WithError(err).Error("error deleting approval items") + return + } + log.Info("Deleted all approval items") + return + + } else if *signatureID != "ALL" { + log.Infof("Migrating approval items for signature : %s", *signatureID) + signature, err := signatureRepo.GetItemSignature(context.Background(), *signatureID) + if err != nil { + log.WithFields(f).WithError(err).Errorf("error fetching signature : %s", *signatureID) + return + } + log.WithFields(f).Debugf("Processing signature : %+v", signature) + err = updateApprovalsTable(signature) + if err != nil { + log.WithFields(f).WithError(err).Errorf("error updating approvals table for signature : %s", *signatureID) + return + } + log.Infof("batch update %d approvals ", len(toUpdateApprovalItems)) + err = approvalRepo.BatchAddApprovalList(toUpdateApprovalItems) + if err != nil { + log.WithFields(f).WithError(err).Error("error adding approval items") + return + } + return + } + + log.Info("Fetching all ccla signatures...") + cclaSignatures, err := signatureRepo.GetCCLASignatures(context.Background(), &signed, &approved) + if err != nil { + log.Fatalf("Error fetching ccla signatures : %v", err) + } + log.Infof("Fetched %d ccla signatures", len(cclaSignatures)) + + log.WithFields(f).Debugf("Fetching events by type : %s", events.ClaApprovalListUpdated) + eventsByType, err = eventsRepo.GetEventsByType(events.ClaApprovalListUpdated, 100) + + if err != nil { + log.WithFields(f).WithError(err).Errorf("error fetching events by type : %s", events.ClaApprovalListUpdated) + return + } + + var wg sync.WaitGroup + + for _, cclaSignature := range cclaSignatures { + wg.Add(1) + go func(signature *signatures.ItemSignature) { + defer wg.Done() + log.WithFields(f).Debugf("Processing company : %s, project : %s", signature.SignatureReferenceName, signature.SignatureProjectID) + updateErr := updateApprovalsTable(signature) + if updateErr != nil { + log.WithFields(f).Warnf("Error updating approvals table for signature : %s, error: %v", signature.SignatureID, updateErr) + } + }(cclaSignature) + } + wg.Wait() + log.WithFields(f).Infof("batch update %d approvals ", len(toUpdateApprovalItems)) + err = approvalRepo.BatchAddApprovalList(toUpdateApprovalItems) + if err != nil { + log.WithFields(f).WithError(err).Error("error adding approval items") + return + } + +} + +func getSearchTermEvents(events []*v1Models.Event, searchTerm, companyID, claGroupID string) []*v1Models.Event { + f := logrus.Fields{ + "functionName": "getSearchTermEvents", + "searchTerm": searchTerm, + "companyID": companyID, + } + log.WithFields(f).Debug("searching for events ...") + var result []*v1Models.Event + for _, event := range events { + if strings.Contains(strings.ToLower(event.EventData), strings.ToLower(searchTerm)) && event.EventCompanyID == companyID && event.EventCLAGroupID == claGroupID { + log.WithFields(f).Debugf("found event with search term : %s", searchTerm) + result = append(result, event) + } + } + return result +} + +func updateApprovalsTable(signature *signatures.ItemSignature) error { + f := logrus.Fields{ + "functionName": "updateApprovalsTable", + "signatureID": signature.SignatureID, + "companyName": signature.SignatureReferenceName, + } + log.WithFields(f).Debugf("updating approvals table for signature : %s", signature.SignatureID) + var wg sync.WaitGroup + var errMutex sync.Mutex + var err error + + update := func(approvalList []string, listType string) { + defer wg.Done() + for _, item := range approvalList { + searchIdentifier := "" + switch listType { + case utils.DomainApprovalCriteria: + searchIdentifier = "email address domain" + case utils.EmailApprovalCriteria: + searchIdentifier = "email address" + case utils.GithubUsernameApprovalCriteria: + searchIdentifier = "GitHub username" + case utils.GitlabUsernameApprovalCriteria: + searchIdentifier = "GitLab username" + case utils.GithubOrgApprovalCriteria: + searchIdentifier = "GitHub organization" + case utils.GitlabOrgApprovalCriteria: + searchIdentifier = "GitLab group" + default: + searchIdentifier = "" + } + searchTerm := fmt.Sprintf("%s %s was added to the approval list", searchIdentifier, item) + events := getSearchTermEvents(eventsByType, searchTerm, signature.SignatureReferenceID, signature.SignatureProjectID) + dateAdded := signature.DateModified + + if len(events) > 0 { + latestEvent := getLatestEvent(events) + dateAdded = latestEvent.EventTime + } + + approvalID, approvalErr := uuid.NewV4() + if err != nil { + errMutex.Lock() + err = approvalErr + log.WithFields(f).Warnf("Error creating new UUIDv4, error: %v", err) + errMutex.Unlock() + return + } + currentTime := time.Now().UTC().String() + note := fmt.Sprintf("Approval item added by migration script on %s", currentTime) + approvalItem := approvals.ApprovalItem{ + ApprovalID: approvalID.String(), + SignatureID: signature.SignatureID, + DateCreated: currentTime, + DateModified: currentTime, + ApprovalName: item, + ApprovalCriteria: listType, + CompanyID: signature.SignatureReferenceID, + ProjectID: signature.SignatureProjectID, + ApprovalCompanyName: signature.SignatureReferenceName, + DateAdded: dateAdded, + Note: note, + Active: true, + } + + toUpdateApprovalItemsMutex.Lock() + toUpdateApprovalItems = append(toUpdateApprovalItems, approvalItem) + toUpdateApprovalItemsMutex.Unlock() + } + } + + wg.Add(1) + go update(signature.EmailDomainApprovalList, utils.DomainApprovalCriteria) + + wg.Add(1) + go update(signature.EmailApprovalList, utils.EmailApprovalCriteria) + + wg.Add(1) + go update(signature.GitHubOrgApprovalList, utils.GithubOrgApprovalCriteria) + + wg.Add(1) + go update(signature.GitHubUsernameApprovalList, utils.GithubUsernameApprovalCriteria) + + wg.Add(1) + go update(signature.GitlabOrgApprovalList, utils.GitlabOrgApprovalCriteria) + + wg.Add(1) + go update(signature.GitlabUsernameApprovalList, utils.GitlabUsernameApprovalCriteria) + + wg.Wait() + + return err +} + +func getLatestEvent(events []*v1Models.Event) *v1Models.Event { + var latest *v1Models.Event + var latestTime time.Time + + for _, item := range events { + t, err := utils.ParseDateTime(item.EventTime) + if err != nil { + log.Debugf("Error parsing time: %+v ", err) + continue + } + + if latest == nil || t.After(latestTime) { + latest = item + latestTime = t + } + } + + return latest +} diff --git a/cla-backend-go/cmd/repositories/repositories.go b/cla-backend-go/cmd/repositories/repositories.go index f5fbaf069..2642c6560 100644 --- a/cla-backend-go/cmd/repositories/repositories.go +++ b/cla-backend-go/cmd/repositories/repositories.go @@ -12,10 +12,10 @@ import ( "github.com/LF-Engineering/lfx-kit/auth" "github.com/aws/aws-sdk-go/aws/awsutil" - "github.com/communitybridge/easycla/cla-backend-go/cmd/functional_tests/test_models" - "github.com/communitybridge/easycla/cla-backend-go/gen/v2/models" - log "github.com/communitybridge/easycla/cla-backend-go/logging" "github.com/go-openapi/swag" + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/functional_tests/test_models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/verdverm/frisby" ) @@ -146,8 +146,8 @@ func (t *TestBehaviour) RunGetProtectedBranch(assertBranchProtection *models.Git return } - if response.BranchName == nil || *response.BranchName != "master" { - F.AddError("Get Protected Branch - Default Branch Name expected : master") + if response.BranchName == nil || *response.BranchName != "main" { + F.AddError("Get Protected Branch - Default Branch Name expected : main") } if len(response.StatusChecks) == 0 { @@ -196,7 +196,7 @@ func (t *TestBehaviour) RunUpdateProtectionBranch(msg string, param *models.Gith }) t.RunGetProtectedBranch(&models.GithubRepositoryBranchProtection{ - BranchName: swag.String("master"), + BranchName: swag.String("main"), EnforceAdmin: *param.EnforceAdmin, ProtectionEnabled: true, StatusChecks: param.StatusChecks, diff --git a/cla-backend-go/cmd/repository_project_update/main.go b/cla-backend-go/cmd/repository_project_update/main.go new file mode 100644 index 000000000..c753d3f43 --- /dev/null +++ b/cla-backend-go/cmd/repository_project_update/main.go @@ -0,0 +1,136 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/dynamodb" + "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" + "github.com/aws/aws-sdk-go/service/dynamodb/expression" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" +) + +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var gitHubRepo RepositoryInterface +var stage string + +type RepositoryInterface interface { + UpdateRepository(ctx context.Context, repositoryID string) error + GetDisabledRepositories(ctx context.Context) ([]*Repository, error) +} + +type repo struct { + tableName string + dynamoDBClient *dynamodb.DynamoDB + stage string +} + +type Repository struct { + RepositoryID string `json:"repository_id"` + Enabled bool `json:"enabled"` + ProjectSFID string `json:"project_sfid"` + ParentProjectSFID string `json:"repository_sfdc_id"` +} + +func (repo *repo) UpdateRepository(ctx context.Context, repositoryID string) error { + updateExpression := expression.Remove(expression.Name("project_sfid")).Remove(expression.Name("repository_sfdc_id")) + expr, err := expression.NewBuilder().WithUpdate(updateExpression).Build() + if err != nil { + return err + } + + _, err = repo.dynamoDBClient.UpdateItemWithContext(ctx, &dynamodb.UpdateItemInput{ + TableName: aws.String(repo.tableName), + Key: map[string]*dynamodb.AttributeValue{ + "repository_id": { + S: aws.String(repositoryID), + }, + }, + UpdateExpression: expr.Update(), + ExpressionAttributeNames: expr.Names(), + }) + if err != nil { + return err + } + + log.Debugf("Updated repository: %s", repositoryID) + + return nil +} + +func (repo *repo) GetDisabledRepositories(ctx context.Context) ([]*Repository, error) { + builder := expression.NewBuilder() + filter := expression.Name("enabled").Equal(expression.Value(false)).And(expression.Name("project_sfid").AttributeExists()).And(expression.Name("repository_sfdc_id").AttributeExists()) + builder = builder.WithFilter(filter) + expr, err := builder.Build() + if err != nil { + return nil, err + } + result, err := repo.dynamoDBClient.ScanWithContext(ctx, &dynamodb.ScanInput{ + TableName: aws.String(repo.tableName), + FilterExpression: expr.Filter(), + ExpressionAttributeNames: expr.Names(), + ExpressionAttributeValues: expr.Values(), + }) + + if err != nil { + return nil, err + } + + if len(result.Items) == 0 { + log.Warn("No disabled repositories found") + return nil, nil + } else { + log.Debugf("Found %d disabled repositories", len(result.Items)) + } + + var out []*Repository + err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &out) + if err != nil { + return nil, err + } + + return out, nil +} + +func NewRepository(awsSession *session.Session, stage string) RepositoryInterface { + return &repo{ + tableName: fmt.Sprintf("cla-%s-repositories", stage), + dynamoDBClient: dynamodb.New(awsSession), + stage: stage, + } +} + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + log.Infof("STAGE set to %s\n", stage) + + gitHubRepo = NewRepository(awsSession, stage) +} + +func main() { + log.Debugf("Getting disabled repositories that have project details...") + context := context.Background() + disabledRepos, err := gitHubRepo.GetDisabledRepositories(context) + if err != nil { + log.Fatalf("Unable to get disabled repositories, error: %v", err) + } + log.Debugf("disabled repositories with existing project details: %v", disabledRepos) + for _, repo := range disabledRepos { + log.Debugf("Updating repository: %+v", *repo) + err := gitHubRepo.UpdateRepository(context, repo.RepositoryID) + if err != nil { + log.Fatalf("Unable to update repository: %s, error: %v", repo.RepositoryID, err) + } + } +} diff --git a/cla-backend-go/cmd/response_metrics.go b/cla-backend-go/cmd/response_metrics.go index 8bb3c07ab..98eee4e7c 100644 --- a/cla-backend-go/cmd/response_metrics.go +++ b/cla-backend-go/cmd/response_metrics.go @@ -4,9 +4,10 @@ package cmd import ( + "sync" "time" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" ) // responseMetrics is a small structure for keeping track of the request metrics @@ -18,32 +19,36 @@ type responseMetrics struct { expire time.Time } -var reqMap = make(map[string]*responseMetrics, 5) +var reqMap sync.Map // requestStart holds the request ID, method and timing information in a small structure func requestStart(reqID, method string) { now, _ := utils.CurrentTime() - reqMap[reqID] = &responseMetrics{ + rm := &responseMetrics{ reqID: reqID, method: method, start: now, elapsed: 0, expire: now.Add(time.Minute * 5), } + reqMap.Store(reqID, rm) } // getRequestMetrics returns the response metrics based on the request id value func getRequestMetrics(reqID string) *responseMetrics { - if x, found := reqMap[reqID]; found { + if val, found := reqMap.Load(reqID); found { + rm, ok := val.(*responseMetrics) + if !ok { + return nil + } now, _ := utils.CurrentTime() - x.elapsed = now.Sub(x.start) - return x + rm.elapsed = now.Sub(rm.start) + return rm } - return nil } // clearRequestMetrics removes the request from the map func clearRequestMetrics(reqID string) { - delete(reqMap, reqID) + reqMap.Delete(reqID) } diff --git a/cla-backend-go/cmd/root.go b/cla-backend-go/cmd/root.go index 5aab0d7d5..7f52b6c81 100644 --- a/cla-backend-go/cmd/root.go +++ b/cla-backend-go/cmd/root.go @@ -8,8 +8,8 @@ import ( "fmt" "os" - ini "github.com/communitybridge/easycla/cla-backend-go/init" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + ini "github.com/linuxfoundation/easycla/cla-backend-go/init" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/spf13/cobra" "github.com/spf13/viper" ) diff --git a/cla-backend-go/cmd/s3_upload/main.go b/cla-backend-go/cmd/s3_upload/main.go new file mode 100644 index 000000000..ca47cded8 --- /dev/null +++ b/cla-backend-go/cmd/s3_upload/main.go @@ -0,0 +1,374 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "encoding/csv" + "encoding/xml" + "strings" + "sync" + + "flag" + + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + + "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/sign" + + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + + "github.com/sirupsen/logrus" +) + +var stage string +var signatureRepo signatures.SignatureRepository +var awsSession = session.Must(session.NewSession(&aws.Config{})) +var companyRepo company.IRepository +var usersRepo users.UserRepository + +var signService sign.Service +var githubOrgService github_organizations.Service +var report []ReportData +var failed int = 0 +var success int = 0 + +func init() { + stage = os.Getenv("STAGE") + if stage == "" { + log.Fatal("STAGE environment variable not set") + } + + companyRepo = company.NewRepository(awsSession, stage) + usersRepo = users.NewRepository(awsSession, stage) + signatureRepo = signatures.NewRepository(awsSession, stage, companyRepo, usersRepo, nil, nil, nil, nil, nil) + githubOrgService = github_organizations.Service{} + configFile, err := config.LoadConfig("", awsSession, stage) + if err != nil { + log.Fatal(err) + } + signService = sign.NewService("", "", companyRepo, nil, nil, nil, nil, configFile.DocuSignPrivateKey, nil, nil, nil, nil, githubOrgService, nil, "", "", nil, nil, nil, nil, nil) + // projectRepo = repository.NewRepository(awsSession, stage, nil, nil, nil) + utils.SetS3Storage(awsSession, configFile.SignatureFilesBucket) +} + +const ( + // Approved Flag + Approved = true + // Signed Flag + Signed = true + + Failed = "failed" + Success = "success" + DocumentUploaded = "Document uploaded successfully" +) + +type ReportData struct { + SignatureID string + ProjectID string + ReferenceID string + ReferenceName string + EnvelopeID string + DocumentID string + Comment string + Status string +} + +type APIErrorResponse struct { + ErrorCode string `json:"errorCode"` + Message string `json:"message"` +} + +func main() { // nolint + ctx := context.Background() + f := logrus.Fields{ + "functionName": "main", + } + // var toUpdate []*signatures.ItemSignature + + dryRun := flag.Bool("dry-run", false, "dry run mode") + folder := flag.String("folder", "", "folder to upload the s3 documents") + meta := flag.String("meta", "", "meta data to upload the s3 documents") + + flag.Parse() + + // Fetch all the signatures from 2024-02-01T00:00:00.000Z + startDate := "2024-02-01T00:00:00.000Z" + + if dryRun != nil && *dryRun { + log.WithFields(f).Debug("dry-run mode enabled") + } + + if folder != nil && *folder != "" && meta != nil && *meta != "" { + log.WithFields(f).Debugf("folder: %s, meta: %s", *folder, *meta) + // var metaMap map[string]string + + // Read csv file + file, err := os.Open(*meta) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem opening meta file") + return + } + + reader := csv.NewReader(file) + records, err := reader.ReadAll() + + count := len(records) + log.WithFields(f).Debugf("processing %d records", count) + + passed := 0 + + if err != nil { + log.WithFields(f).WithError(err).Warn("problem reading meta file") + return + } + + var wg sync.WaitGroup + + // Limit the number of concurrent uploads + semaphore := make(chan struct{}, 5) + + for _, record := range records { + wg.Add(1) + semaphore <- struct{}{} + go func(record []string) { + defer wg.Done() + defer func() { <-semaphore }() + fileName := record[0] + envelopeID := record[1] + signatureID := record[2] + projectID := record[3] + referenceID := record[4] + log.WithFields(f).Debugf("uploading file: %s, envelopeID: %s, signatureID: %s, projectID: %s, referenceID: %s", fileName, envelopeID, signatureID, projectID, referenceID) + // Upload the file + file, err := os.Open(*folder + "/" + fileName) // nolint + if err != nil { + log.WithFields(f).WithError(err).Warn("problem opening file") + failed++ + return + } + + if dryRun != nil && *dryRun { + log.WithFields(f).Debugf("dry-run mode enabled, skipping file upload: %s", fileName) + return + } + + // Upload the document + log.WithFields(f).Debugf("uploading document for signature...: %s", signatureID) + + err = utils.UploadFileToS3(file, projectID, utils.ClaTypeICLA, referenceID, signatureID) + + if err != nil { + log.WithFields(f).WithError(err).Warn("problem uploading file") + failed++ + return + } + passed++ + + log.WithFields(f).Debugf("document uploaded for signature: %s", signatureID) + + }(record) + } + + wg.Wait() + + log.WithFields(f).Debug("completed processing files") + + log.WithFields(f).Debugf("total: %d, passed: %d, failed: %d", count, passed, failed) + return + } + + iclaSignatures, err := signatureRepo.GetICLAByDate(ctx, startDate) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem fetching ICLA signatures") + return + } + + log.WithFields(f).Debugf("processing %d ICLA signatures", len(iclaSignatures)) + toUpload := make([]signatures.ItemSignature, 0) + + var wg sync.WaitGroup + semaphore := make(chan struct{}, 20) + + for _, icla := range iclaSignatures { + wg.Add(1) + semaphore <- struct{}{} + go func(sig signatures.ItemSignature) { + defer wg.Done() + defer func() { <-semaphore }() + key := strings.Join([]string{"contract-group", sig.SignatureProjectID, utils.ClaTypeICLA, sig.SignatureReferenceID, sig.SignatureID}, "/") + ".pdf" + fileExists, fileErr := utils.DocumentExists(key) + if fileErr != nil { + log.WithFields(f).WithError(fileErr).Debugf("unable to check s3 key : %s", key) + return + } + if !fileExists { + log.WithFields(f).Debugf("document is not uploaded for key: %s", key) + toUpload = append(toUpload, sig) + } else { + log.WithFields(f).Debugf("key: %s exists", key) + } + }(icla) + + } + + log.WithFields(f).Debugf("checking icla signatures from :%s", startDate) + wg.Wait() + log.WithFields(f).Debugf("To upload %d icla signatures: ", len(toUpload)) + + // Upload the documents + for _, icla := range toUpload { + wg.Add(1) + semaphore <- struct{}{} + go func(sig signatures.ItemSignature) { + defer wg.Done() + + var documentID string + + reportData := ReportData{ + SignatureID: sig.SignatureID, + ProjectID: sig.SignatureProjectID, + ReferenceID: sig.SignatureReferenceID, + ReferenceName: sig.SignatureReferenceName, + EnvelopeID: sig.SignatureEnvelopeID, + } + + // get the document id + var info sign.DocuSignEnvelopeInformation + + if sig.UserDocusignRawXML == "" { + log.WithFields(f).Debugf("no raw xml found for signature: %s", sig.SignatureID) + reportData.Comment = "No raw xml found" + // Fetch documentID + documents, docErr := signService.GetEnvelopeDocuments(ctx, sig.SignatureEnvelopeID) + if docErr != nil { + log.WithFields(f).WithError(err).Debugf("unable to get documents for signature: %s", sig.SignatureID) + reportData.Comment = docErr.Error() + reportData.Status = Failed + report = append(report, reportData) + failed++ + return + } + if len(documents) == 0 { + log.WithFields(f).Debugf("no documents found for signature: %s", sig.SignatureID) + reportData.Comment = "No documents found" + reportData.Status = Failed + report = append(report, reportData) + failed++ + return + } + documentID = documents[0].DocumentId + log.WithFields(f).Debugf("document id fetched from docusign: %s", documentID) + } else { + err = xml.Unmarshal([]byte(sig.UserDocusignRawXML), &info) + if err != nil { + log.WithFields(f).WithError(err).Debugf("unable to unmarshal xml for signature: %s", sig.SignatureID) + reportData.Comment = err.Error() + reportData.Status = Failed + report = append(report, reportData) + failed++ + return + } + documentID = info.EnvelopeStatus.DocumentStatuses[0].ID + } + + log.WithFields(f).Debugf("document id: %s", documentID) + reportData.DocumentID = documentID + envelopeID := sig.SignatureEnvelopeID + log.WithFields(f).Debugf("envelope id: %s", envelopeID) + + if documentID == "" { + log.WithFields(f).Debugf("no document id found for signature: %s", sig.SignatureID) + reportData.Comment = "No document id found" + reportData.Status = Failed + report = append(report, reportData) + failed++ + return + } + + // get the document + document, docErr := signService.GetSignedDocument(ctx, envelopeID, documentID) + if docErr != nil { + log.WithFields(f).WithError(docErr).Debugf("unable to get document for signature: %s", sig.SignatureID) + reportData.Comment = docErr.Error() + reportData.Status = Failed + report = append(report, reportData) + failed++ + return + } + // upload the document + if dryRun != nil && *dryRun { + log.WithFields(f).Debugf("dry-run mode enabled, skipping document upload for signature: %s", sig.SignatureID) + log.WithFields(f).Debugf("document uploaded for signature: %s", sig.SignatureID) + reportData.Comment = DocumentUploaded + reportData.Status = Success + report = append(report, reportData) + return + } + + log.WithFields(f).Debugf("uploading document for signature...: %s", sig.SignatureID) + err = utils.UploadToS3(document, sig.SignatureProjectID, utils.ClaTypeICLA, sig.SignatureReferenceID, sig.SignatureID) + if err != nil { + log.WithFields(f).WithError(err).Debugf("unable to upload document for signature: %s", sig.SignatureID) + reportData.Comment = err.Error() + reportData.Status = Failed + failed++ + report = append(report, reportData) + return + } + + log.WithFields(f).Debugf("document uploaded for signature: %s", sig.SignatureID) + reportData.Comment = DocumentUploaded + reportData.Status = Success + success++ + + report = append(report, reportData) + + // release the semaphore + <-semaphore + + }(icla) + } + + wg.Wait() + + log.WithFields(f).Debug("completed processing ICLA signatures") + + file, err := os.Create("s3_upload_report.csv") + if err != nil { + log.WithFields(f).WithError(err).Warn("problem creating report file") + return + } + + writer := csv.NewWriter(file) + defer writer.Flush() + + err = writer.Write([]string{"SignatureID", "ProjectID", "ReferenceID", "ReferenceName", "EnvelopeID", "DocumentID", "Comment", "Status"}) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem writing header to report file") + return + } + + for _, data := range report { + // writer.Write([]string{data.SignatureID, data.ProjectID, data.ReferenceID, data.ReferenceName, data.EnvelopeID, data.DocumentID, data.Comment}) + record := []string{data.SignatureID, data.ProjectID, data.ReferenceID, data.ReferenceName, data.EnvelopeID, data.DocumentID, data.Comment, data.Status} + err = writer.Write(record) + if err != nil { + log.WithFields(f).WithError(err).Warn("problem writing record to report file") + } + } + + log.WithFields(f).Debugf("report generated successfully, total: %d, success: %d, failed: %d", len(report), success, failed) + + log.WithFields(f).Debug("report generated successfully") +} diff --git a/cla-backend-go/cmd/server.go b/cla-backend-go/cmd/server.go index 6ac014d60..2dda7f268 100644 --- a/cla-backend-go/cmd/server.go +++ b/cla-backend-go/cmd/server.go @@ -4,8 +4,10 @@ package cmd import ( + "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/url" @@ -14,75 +16,92 @@ import ( "strconv" "strings" - "github.com/communitybridge/easycla/cla-backend-go/v2/dynamo_events" - v2GithubActivity "github.com/communitybridge/easycla/cla-backend-go/v2/github_activity" + "github.com/linuxfoundation/easycla/cla-backend-go/project/repository" + "github.com/linuxfoundation/easycla/cla-backend-go/project/service" + + gitlab_activity "github.com/linuxfoundation/easycla/cla-backend-go/v2/gitlab-activity" + + "github.com/go-openapi/strfmt" + + "github.com/linuxfoundation/easycla/cla-backend-go/v2/gitlab_organizations" + + gitlab "github.com/linuxfoundation/easycla/cla-backend-go/gitlab_api" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/gitlab_sign" + + "github.com/linuxfoundation/easycla/cla-backend-go/emails" + + "github.com/linuxfoundation/easycla/cla-backend-go/v2/dynamo_events" + v2GithubActivity "github.com/linuxfoundation/easycla/cla-backend-go/v2/github_activity" "github.com/gofrs/uuid" - "github.com/communitybridge/easycla/cla-backend-go/approval_list" - "github.com/communitybridge/easycla/cla-backend-go/v2/cla_groups" openapi_runtime "github.com/go-openapi/runtime" + "github.com/linuxfoundation/easycla/cla-backend-go/approval_list" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/projects_cla_groups" + "github.com/linuxfoundation/easycla/cla-backend-go/projects_cla_groups" - "github.com/communitybridge/easycla/cla-backend-go/v2/sign" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/sign" - "github.com/communitybridge/easycla/cla-backend-go/cla_manager" - project_service "github.com/communitybridge/easycla/cla-backend-go/v2/project-service" - user_service "github.com/communitybridge/easycla/cla-backend-go/v2/user-service" + "github.com/linuxfoundation/easycla/cla-backend-go/cla_manager" + project_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/project-service" + user_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/user-service" - acs_service "github.com/communitybridge/easycla/cla-backend-go/v2/acs-service" - organization_service "github.com/communitybridge/easycla/cla-backend-go/v2/organization-service" + acs_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/acs-service" + organization_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" - "github.com/communitybridge/easycla/cla-backend-go/github_organizations" - v2GithubOrganizations "github.com/communitybridge/easycla/cla-backend-go/v2/github_organizations" - "github.com/communitybridge/easycla/cla-backend-go/v2/metrics" + "github.com/linuxfoundation/easycla/cla-backend-go/github_organizations" + v2GithubOrganizations "github.com/linuxfoundation/easycla/cla-backend-go/v2/github_organizations" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/metrics" - "github.com/communitybridge/easycla/cla-backend-go/gerrits" - v2Gerrits "github.com/communitybridge/easycla/cla-backend-go/v2/gerrits" + "github.com/linuxfoundation/easycla/cla-backend-go/gerrits" + v2Gerrits "github.com/linuxfoundation/easycla/cla-backend-go/v2/gerrits" "github.com/aws/aws-sdk-go/service/dynamodb" lfxAuth "github.com/LF-Engineering/lfx-kit/auth" - "github.com/communitybridge/easycla/cla-backend-go/docs" - "github.com/communitybridge/easycla/cla-backend-go/repositories" - "github.com/communitybridge/easycla/cla-backend-go/utils" - v2Docs "github.com/communitybridge/easycla/cla-backend-go/v2/docs" - v2Events "github.com/communitybridge/easycla/cla-backend-go/v2/events" - v2Metrics "github.com/communitybridge/easycla/cla-backend-go/v2/metrics" - v2Repositories "github.com/communitybridge/easycla/cla-backend-go/v2/repositories" - v2Version "github.com/communitybridge/easycla/cla-backend-go/v2/version" - "github.com/communitybridge/easycla/cla-backend-go/version" - - "github.com/communitybridge/easycla/cla-backend-go/events" - - "github.com/communitybridge/easycla/cla-backend-go/project" - v2Project "github.com/communitybridge/easycla/cla-backend-go/v2/project" - - "github.com/communitybridge/easycla/cla-backend-go/users" - - "github.com/communitybridge/easycla/cla-backend-go/signatures" - v2Signatures "github.com/communitybridge/easycla/cla-backend-go/v2/signatures" - - ini "github.com/communitybridge/easycla/cla-backend-go/init" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - - "github.com/communitybridge/easycla/cla-backend-go/auth" - "github.com/communitybridge/easycla/cla-backend-go/company" - "github.com/communitybridge/easycla/cla-backend-go/docraptor" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations" - v2RestAPI "github.com/communitybridge/easycla/cla-backend-go/gen/v2/restapi" - v2Ops "github.com/communitybridge/easycla/cla-backend-go/gen/v2/restapi/operations" - "github.com/communitybridge/easycla/cla-backend-go/github" - "github.com/communitybridge/easycla/cla-backend-go/health" - "github.com/communitybridge/easycla/cla-backend-go/template" - "github.com/communitybridge/easycla/cla-backend-go/user" - v2ClaManager "github.com/communitybridge/easycla/cla-backend-go/v2/cla_manager" - v2Company "github.com/communitybridge/easycla/cla-backend-go/v2/company" - v2Health "github.com/communitybridge/easycla/cla-backend-go/v2/health" - v2Template "github.com/communitybridge/easycla/cla-backend-go/v2/template" + "github.com/linuxfoundation/easycla/cla-backend-go/docs" + v1Repositories "github.com/linuxfoundation/easycla/cla-backend-go/repositories" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + v2Docs "github.com/linuxfoundation/easycla/cla-backend-go/v2/docs" + v2Events "github.com/linuxfoundation/easycla/cla-backend-go/v2/events" + v2Metrics "github.com/linuxfoundation/easycla/cla-backend-go/v2/metrics" + v2Repositories "github.com/linuxfoundation/easycla/cla-backend-go/v2/repositories" + v2Version "github.com/linuxfoundation/easycla/cla-backend-go/v2/version" + "github.com/linuxfoundation/easycla/cla-backend-go/version" + + "github.com/linuxfoundation/easycla/cla-backend-go/events" + + "github.com/linuxfoundation/easycla/cla-backend-go/project" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/approvals" + v2Project "github.com/linuxfoundation/easycla/cla-backend-go/v2/project" + + "github.com/linuxfoundation/easycla/cla-backend-go/users" + + "github.com/linuxfoundation/easycla/cla-backend-go/signatures" + v2Signatures "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures" + + ini "github.com/linuxfoundation/easycla/cla-backend-go/init" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + + "github.com/linuxfoundation/easycla/cla-backend-go/auth" + v1Company "github.com/linuxfoundation/easycla/cla-backend-go/company" + "github.com/linuxfoundation/easycla/cla-backend-go/docraptor" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations" + v2RestAPI "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi" + v2Ops "github.com/linuxfoundation/easycla/cla-backend-go/gen/v2/restapi/operations" + "github.com/linuxfoundation/easycla/cla-backend-go/github" + "github.com/linuxfoundation/easycla/cla-backend-go/health" + "github.com/linuxfoundation/easycla/cla-backend-go/template" + "github.com/linuxfoundation/easycla/cla-backend-go/user" + v2ClaManager "github.com/linuxfoundation/easycla/cla-backend-go/v2/cla_manager" + v2Company "github.com/linuxfoundation/easycla/cla-backend-go/v2/company" + v2CurrentUser "github.com/linuxfoundation/easycla/cla-backend-go/v2/current_user" + v2Health "github.com/linuxfoundation/easycla/cla-backend-go/v2/health" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/store" + v2Template "github.com/linuxfoundation/easycla/cla-backend-go/v2/template" "github.com/go-openapi/loads" "github.com/rs/cors" @@ -120,8 +139,17 @@ func init() { type combinedRepo struct { users.UserRepository - company.IRepository - project.ProjectRepository + v1Company.IRepository + repository.ProjectRepository + projects_cla_groups.Repository +} + +// in cmd/server.go (top-level imports already use logrus) +func apiPathLogger(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + log.Infof("LG:api-request-path:%s", r.URL.Path) + next.ServeHTTP(w, r) + }) } // server function called by environment specific server functions @@ -215,75 +243,105 @@ func server(localMode bool) http.Handler { log.WithFields(f).WithError(err).Panic("unable to setup docraptor client") } + // LG: to test with manual tokens + customClaimUsername := os.Getenv("AUTH0_USERNAME_CLAIM_CLI") + if customClaimUsername != "" { + configFile.Auth0.UsernameClaim = customClaimUsername + } + nameClaimName := os.Getenv("AUTH0_NAME_CLAIM_CLI") + if nameClaimName == "" { + nameClaimName = "name" + } + emailClaimName := os.Getenv("AUTH0_EMAIL_CLAIM_CLI") + if emailClaimName == "" { + emailClaimName = "email" + } authValidator, err := auth.NewAuthValidator( configFile.Auth0.Domain, configFile.Auth0.ClientID, configFile.Auth0.UsernameClaim, + nameClaimName, + emailClaimName, configFile.Auth0.Algorithm) if err != nil { logrus.Panic(err) } - github.Init(configFile.Github.AppID, configFile.Github.AppPrivateKey, configFile.Github.AccessToken) + // initialize github + github.Init(configFile.GitHub.AppID, configFile.GitHub.AppPrivateKey, configFile.GitHub.AccessToken) + // initialize gitlab + gitlabApp := gitlab.Init(configFile.Gitlab.AppClientID, configFile.Gitlab.AppClientSecret, configFile.Gitlab.AppPrivateKey) // Our backend repository handlers userRepo := user.NewDynamoRepository(awsSession, stage) usersRepo := users.NewRepository(awsSession, stage) - repositoriesRepo := repositories.NewRepository(awsSession, stage) + gitV1Repository := v1Repositories.NewRepository(awsSession, stage) + gitV2Repository := v2Repositories.NewRepository(awsSession, stage) gerritRepo := gerrits.NewRepository(awsSession, stage) templateRepo := template.NewRepository(awsSession, stage) approvalListRepo := approval_list.NewRepository(awsSession, stage) - companyRepo := company.NewRepository(awsSession, stage) - signaturesRepo := signatures.NewRepository(awsSession, stage, companyRepo, usersRepo) - projectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) - projectRepo := project.NewRepository(awsSession, stage, repositoriesRepo, gerritRepo, projectClaGroupRepo) + v1CompanyRepo := v1Company.NewRepository(awsSession, stage) eventsRepo := events.NewRepository(awsSession, stage) - metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, projectClaGroupRepo) + v1ProjectClaGroupRepo := projects_cla_groups.NewRepository(awsSession, stage) + v1CLAGroupRepo := repository.NewRepository(awsSession, stage, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo) + metricsRepo := metrics.NewRepository(awsSession, stage, configFile.APIGatewayURL, v1ProjectClaGroupRepo) githubOrganizationsRepo := github_organizations.NewRepository(awsSession, stage) + gitlabOrganizationRepo := gitlab_organizations.NewRepository(awsSession, stage) claManagerReqRepo := cla_manager.NewRepository(awsSession, stage) + storeRepository := store.NewRepository(awsSession, stage) + approvalsRepo := approvals.NewRepository(stage, awsSession, fmt.Sprintf("cla-%s-approvals", stage)) // Our service layer handlers eventsService := events.NewService(eventsRepo, combinedRepo{ usersRepo, - companyRepo, - projectRepo, + v1CompanyRepo, + v1CLAGroupRepo, + v1ProjectClaGroupRepo, }) + gerritService := gerrits.NewService(gerritRepo) + + // Signature repository handler + signaturesRepo := signatures.NewRepository(awsSession, stage, v1CompanyRepo, usersRepo, eventsService, gitV1Repository, githubOrganizationsRepo, gerritService, approvalsRepo) + // Initialize the external platform services - these are external APIs that // we download the swagger specification, generate the models, and have //client helper functions - user_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) - project_service.InitClient(configFile.APIGatewayURL) - organization_service.InitClient(configFile.APIGatewayURL, eventsService) - acs_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) + user_service.InitClient(configFile.PlatformAPIGatewayURL, configFile.AcsAPIKey) + project_service.InitClient(configFile.PlatformAPIGatewayURL) + organization_service.InitClient(configFile.PlatformAPIGatewayURL, eventsService) + acs_service.InitClient(configFile.PlatformAPIGatewayURL, configFile.AcsAPIKey) + v1ProjectClaGroupService := projects_cla_groups.NewService(v1ProjectClaGroupRepo) usersService := users.NewService(usersRepo, eventsService) healthService := health.New(Version, Commit, Branch, BuildDate) templateService := template.NewService(stage, templateRepo, docraptorClient, awsSession) - projectService := project.NewService(projectRepo, repositoriesRepo, gerritRepo, projectClaGroupRepo, usersRepo) - v2ProjectService := v2Project.NewService(projectService, projectRepo, projectClaGroupRepo) - companyService := company.NewService(companyRepo, configFile.CorporateConsoleURL, userRepo, usersService) - v2CompanyService := v2Company.NewService(companyService, signaturesRepo, projectRepo, usersRepo, companyRepo, projectClaGroupRepo, eventsService) - v2SignService := sign.NewService(configFile.ClaV1ApiURL, companyRepo, projectRepo, projectClaGroupRepo, companyService) - signaturesService := signatures.NewService(signaturesRepo, companyService, usersService, eventsService, githubOrgValidation) - v2SignatureService := v2Signatures.NewService(awsSession, configFile.SignatureFilesBucket, projectService, companyService, signaturesService, projectClaGroupRepo) - v1ClaManagerService := cla_manager.NewService(claManagerReqRepo, companyService, projectService, usersService, signaturesService, eventsService, configFile.CorporateConsoleURL) - repositoriesService := repositories.NewService(repositoriesRepo, githubOrganizationsRepo, projectClaGroupRepo) - v2RepositoriesService := v2Repositories.NewService(repositoriesRepo, projectClaGroupRepo, githubOrganizationsRepo) - v2ClaManagerService := v2ClaManager.NewService(companyService, projectService, v1ClaManagerService, usersService, repositoriesService, v2CompanyService, eventsService, projectClaGroupRepo) - approvalListService := approval_list.NewService(approvalListRepo, usersRepo, companyRepo, projectRepo, signaturesRepo, configFile.CorporateConsoleURL, http.DefaultClient) + v1ProjectService := service.NewService(v1CLAGroupRepo, gitV1Repository, gerritRepo, v1ProjectClaGroupRepo, usersRepo) + emailTemplateService := emails.NewEmailTemplateService(v1CLAGroupRepo, v1ProjectClaGroupRepo, v1ProjectService, configFile.CorporateConsoleV1URL, configFile.CorporateConsoleV2URL) + emailService := emails.NewService(emailTemplateService, v1ProjectService) + v2ProjectService := v2Project.NewService(v1ProjectService, v1CLAGroupRepo, v1ProjectClaGroupRepo) + v1CompanyService := v1Company.NewService(v1CompanyRepo, configFile.CorporateConsoleV1URL, userRepo, usersService) + v2CompanyService := v2Company.NewService(v1CompanyService, signaturesRepo, v1CLAGroupRepo, usersRepo, v1CompanyRepo, v1ProjectClaGroupRepo, eventsService) + v2CurrentUserService := v2CurrentUser.NewService() + + v1RepositoriesService := v1Repositories.NewService(gitV1Repository, githubOrganizationsRepo, v1ProjectClaGroupRepo) + v2RepositoriesService := v2Repositories.NewService(gitV1Repository, gitV2Repository, v1ProjectClaGroupRepo, githubOrganizationsRepo, gitlabOrganizationRepo, eventsService) + githubOrganizationsService := github_organizations.NewService(githubOrganizationsRepo, gitV1Repository, v1ProjectClaGroupRepo) + gitlabOrganizationsService := gitlab_organizations.NewService(gitlabOrganizationRepo, v2RepositoriesService, v1ProjectClaGroupRepo, storeRepository, usersService, signaturesRepo, v1CompanyRepo) + v1SignaturesService := signatures.NewService(signaturesRepo, v1CompanyService, usersService, eventsService, githubOrgValidation, v1RepositoriesService, githubOrganizationsService, v1ProjectService, gitlabApp, configFile.ClaV1ApiURL, configFile.CLALandingPage, configFile.CLALogoURL) + v2SignatureService := v2Signatures.NewService(awsSession, configFile.SignatureFilesBucket, v1ProjectService, v1CompanyService, v1SignaturesService, v1ProjectClaGroupRepo, signaturesRepo, usersService, approvalsRepo) + v1ClaManagerService := cla_manager.NewService(claManagerReqRepo, v1ProjectClaGroupRepo, v1CompanyService, v1ProjectService, usersService, v1SignaturesService, eventsService, emailTemplateService, configFile.CorporateConsoleV1URL) + v2ClaManagerService := v2ClaManager.NewService(emailTemplateService, v1CompanyService, v1ProjectService, v1ClaManagerService, usersService, v1RepositoriesService, v2CompanyService, eventsService, v1ProjectClaGroupRepo) + v1ApprovalListService := approval_list.NewService(approvalListRepo, v1ProjectClaGroupRepo, v1ProjectService, usersRepo, v1CompanyRepo, v1CLAGroupRepo, signaturesRepo, emailTemplateService, configFile.CorporateConsoleV2URL, http.DefaultClient) authorizer := auth.NewAuthorizer(authValidator, userRepo) - v2MetricsService := metrics.NewService(metricsRepo, projectClaGroupRepo) - githubOrganizationsService := github_organizations.NewService(githubOrganizationsRepo, repositoriesRepo, projectClaGroupRepo) - v2GithubOrganizationsService := v2GithubOrganizations.NewService(githubOrganizationsRepo, repositoriesRepo, projectClaGroupRepo) - autoEnableService := dynamo_events.NewAutoEnableService(repositoriesService, repositoriesRepo, githubOrganizationsRepo, projectClaGroupRepo, projectService) - v2GithubActivityService := v2GithubActivity.NewService(repositoriesRepo, eventsService, autoEnableService) - gerritService := gerrits.NewService(gerritRepo, &gerrits.LFGroup{ - LfBaseURL: configFile.LFGroup.ClientURL, - ClientID: configFile.LFGroup.ClientID, - ClientSecret: configFile.LFGroup.ClientSecret, - RefreshToken: configFile.LFGroup.RefreshToken, - }) - v2ClaGroupService := cla_groups.NewService(projectService, templateService, projectClaGroupRepo, v1ClaManagerService, signaturesService, metricsRepo, gerritService, repositoriesService, eventsService) + v2MetricsService := metrics.NewService(metricsRepo, v1ProjectClaGroupRepo) + gitlabActivityService := gitlab_activity.NewService(gitV1Repository, gitV2Repository, usersRepo, signaturesRepo, v1ProjectClaGroupRepo, v1CompanyRepo, signaturesRepo, gitlabOrganizationsService) + gitlabSignService := gitlab_sign.NewService(v2RepositoriesService, usersService, storeRepository, gitlabApp, gitlabOrganizationsService) + v2GithubOrganizationsService := v2GithubOrganizations.NewService(githubOrganizationsRepo, gitV1Repository, v1ProjectClaGroupRepo, githubOrganizationsService) + autoEnableService := dynamo_events.NewAutoEnableService(v1RepositoriesService, gitV1Repository, githubOrganizationsRepo, v1ProjectClaGroupRepo, v1ProjectService) + v2GithubActivityService := v2GithubActivity.NewService(gitV1Repository, githubOrganizationsRepo, eventsService, autoEnableService, emailService) + + v2ClaGroupService := cla_groups.NewService(v1ProjectService, templateService, v1ProjectClaGroupRepo, v1ClaManagerService, v1SignaturesService, metricsRepo, gerritService, v1RepositoriesService, eventsService) + v2SignService := sign.NewService(configFile.ClaAPIV4Base, configFile.ClaV1ApiURL, v1CompanyRepo, v1CLAGroupRepo, v1ProjectClaGroupRepo, v1CompanyService, v2ClaGroupService, configFile.DocuSignPrivateKey, usersService, v1SignaturesService, storeRepository, v1RepositoriesService, githubOrganizationsService, gitlabOrganizationsService, configFile.CLALandingPage, configFile.CLALogoURL, emailService, eventsService, gitlabActivityService, gitlabApp, gerritService) sessionStore, err := dynastore.New(dynastore.Path("/"), dynastore.HTTPOnly(), dynastore.TableName(configFile.SessionStoreTableName), dynastore.DynamoDB(dynamodb.New(awsSession))) if err != nil { @@ -298,40 +356,49 @@ func server(localMode bool) http.Handler { // Setup our API handlers users.Configure(api, usersService, eventsService) - project.Configure(api, projectService, eventsService, gerritService, repositoriesService, signaturesService) - v2Project.Configure(v2API, projectService, v2ProjectService, eventsService) + project.Configure(api, v1ProjectService, eventsService, gerritService, v1RepositoriesService, v1SignaturesService) + v2Project.Configure(v2API, v1ProjectService, v2ProjectService, eventsService, v1ProjectClaGroupService, v2RepositoriesService, gerritService) health.Configure(api, healthService) v2Health.Configure(v2API, healthService) template.Configure(api, templateService, eventsService) - v2Template.Configure(v2API, templateService, eventsService) - github.Configure(api, configFile.Github.ClientID, configFile.Github.ClientSecret, configFile.Github.AccessToken, sessionStore) - signatures.Configure(api, signaturesService, sessionStore, eventsService) - v2Signatures.Configure(v2API, projectService, projectRepo, companyService, signaturesService, sessionStore, eventsService, v2SignatureService, projectClaGroupRepo) - approval_list.Configure(api, approvalListService, sessionStore, signaturesService, eventsService) - company.Configure(api, companyService, usersService, companyUserValidation, eventsService) + v2Template.Configure(v2API, templateService, v1ProjectClaGroupService, eventsService) + github.Configure(api, configFile.GitHub.ClientID, configFile.GitHub.ClientSecret, configFile.GitHub.AccessToken, sessionStore) + signatures.Configure(api, v1SignaturesService, sessionStore, eventsService) + v2Signatures.Configure(v2API, v1ProjectService, v1CLAGroupRepo, v1CompanyService, v1SignaturesService, sessionStore, eventsService, v2SignatureService, v1ProjectClaGroupRepo) + approval_list.Configure(api, v1ApprovalListService, sessionStore, v1SignaturesService, eventsService) + v1Company.Configure(api, v1CompanyService, usersService, companyUserValidation, eventsService) docs.Configure(api) v2Docs.Configure(v2API) version.Configure(api, Version, Commit, Branch, BuildDate) v2Version.Configure(v2API, Version, Commit, Branch, BuildDate) events.Configure(api, eventsService) - v2Events.Configure(v2API, eventsService, companyRepo, projectClaGroupRepo) - v2Metrics.Configure(v2API, v2MetricsService, companyRepo) + v2Events.Configure(v2API, eventsService, v1CompanyRepo, v1ProjectClaGroupRepo, v1ProjectService) + v2Metrics.Configure(v2API, v2MetricsService, v1CompanyRepo) github_organizations.Configure(api, githubOrganizationsService, eventsService) v2GithubOrganizations.Configure(v2API, v2GithubOrganizationsService, eventsService) - repositories.Configure(api, repositoriesService, eventsService) + gitlab_organizations.Configure(v2API, gitlabOrganizationsService, eventsService, sessionStore, configFile.CLAContributorv2Base) + gitlab_sign.Configure(v2API, gitlabSignService, eventsService, configFile.CLAContributorv2Base, sessionStore) + gitlab_activity.Configure(v2API, gitlabActivityService, gitlabOrganizationsService, eventsService, gitlabApp, gitlabSignService, configFile.CLAContributorv2Base, sessionStore) + v1Repositories.Configure(api, v1RepositoriesService, eventsService) v2Repositories.Configure(v2API, v2RepositoriesService, eventsService) - gerrits.Configure(api, gerritService, projectService, eventsService) - v2Gerrits.Configure(v2API, gerritService, projectService, eventsService, projectClaGroupRepo) - v2Company.Configure(v2API, v2CompanyService, companyRepo, projectClaGroupRepo, configFile.LFXPortalURL, configFile.CorporateConsoleURL) - cla_manager.Configure(api, v1ClaManagerService, companyService, projectService, usersService, signaturesService, eventsService, configFile.CorporateConsoleURL) - v2ClaManager.Configure(v2API, v2ClaManagerService, configFile.LFXPortalURL, projectClaGroupRepo, userRepo) - sign.Configure(v2API, v2SignService) - cla_groups.Configure(v2API, v2ClaGroupService, projectService, projectClaGroupRepo, eventsService) + gerrits.Configure(api, gerritService, v1ProjectService, eventsService) + v2Gerrits.Configure(v2API, gerritService, v1ProjectService, eventsService, v1ProjectClaGroupRepo) + v2Company.Configure(v2API, v2CompanyService, v1ProjectClaGroupRepo, configFile.LFXPortalURL, configFile.CorporateConsoleV1URL) + v2CurrentUser.Configure(v2API, v2CurrentUserService) + cla_manager.Configure(api, v1ClaManagerService, v1CompanyService, v1ProjectService, usersService, v1SignaturesService, eventsService, emailTemplateService) + v2ClaManager.Configure(v2API, v2ClaManagerService, v1CompanyService, configFile.LFXPortalURL, configFile.CorporateConsoleV2URL, v1ProjectClaGroupRepo, userRepo) + cla_groups.Configure(v2API, v2ClaGroupService, v1ProjectService, v1ProjectClaGroupRepo, eventsService) + sign.Configure(v2API, v2SignService, usersService) v2GithubActivity.Configure(v2API, v2GithubActivityService) + v2API.AddMiddlewareFor("POST", "/signed/individual/{installation_id}/{github_repository_id}/{change_request_id}", sign.DocusignMiddleware) + v2API.AddMiddlewareFor("POST", "/signed/corporate/{project_id}/{company_id}", sign.CCLADocusignMiddleware) + v2API.AddMiddlewareFor("POST", "/signed/gitlab/individual/{user_id}/{organization_id}/{gitlab_repository_id}/{merge_request_id}", sign.DocusignMiddleware) + v2API.AddMiddlewareFor("POST", "/signed/gerrit/individual/{user_id}", sign.DocusignMiddleware) + userCreaterMiddleware := func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - createUserFromRequest(authorizer, usersService, eventsService, r) + r = createUserFromRequest(authorizer, usersService, eventsService, r) next.ServeHTTP(w, r) }) } @@ -339,7 +406,7 @@ func server(localMode bool) http.Handler { // The middleware configuration is for the handler executors. These do not apply to the swagger.json document. // The middleware executes after routing but before authentication, binding and validation middlewareSetupfunc := func(handler http.Handler) http.Handler { - return setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler))) + return apiPathLogger(setRequestIDHandler(responseLoggingMiddleware(userCreaterMiddleware(handler)))) } v2API.CsvProducer = openapi_runtime.ProducerFunc(func(w io.Writer, data interface{}) error { @@ -570,59 +637,85 @@ func responseLoggingMiddleware(next http.Handler) http.Handler { // create user form http authorization token // this function creates user if user does not exist and token is valid -func createUserFromRequest(authorizer auth.Authorizer, usersService users.Service, eventsService events.Service, r *http.Request) { +func createUserFromRequest(authorizer auth.Authorizer, usersService users.Service, eventsService events.Service, r *http.Request) *http.Request { f := logrus.Fields{ "functionName": "cmd.createUserFromRequest", } bToken := r.Header.Get("Authorization") if bToken == "" { - return + return r } t := strings.Split(bToken, " ") if len(t) != 2 { log.WithFields(f).Warn("parsing of authorization header failed - expected two values separated by a space") - return + return r } // parse user from the auth token claUser, err := authorizer.SecurityAuth(t[1], []string{}) if err != nil { log.WithFields(f).WithError(err).Warn("parsing failed") - return + return r } + f["claUserName"] = claUser.Name + f["claUserID"] = claUser.UserID + f["claUserLFUsername"] = claUser.LFUsername + f["claUserLFEmail"] = claUser.LFEmail + f["claUserEmails"] = strings.Join(claUser.Emails, ",") + + // only needed if API called is /v4/user-from-token + needToStoreUser := r.URL.Path == "/v4/user-from-token" // search if user exist in database by username userModel, err := usersService.GetUserByLFUserName(claUser.LFUsername) if err != nil { - log.WithFields(f).WithError(err).Warn("searching user by lf-username failed") - return + if _, ok := err.(*utils.UserNotFound); ok { + log.WithFields(f).Debug("unable to locate user by lf-email") + } else { + log.WithFields(f).WithError(err).Warn("searching user by lf-username failed") + return r + } } + // If found - just return if userModel != nil { - return + if !needToStoreUser { + return r + } + ctx := context.WithValue(r.Context(), "user", userModel) // nolint + return r.WithContext(ctx) } // search if user exist in database by username userModel, err = usersService.GetUserByEmail(claUser.LFEmail) if err != nil { - log.WithFields(f).WithError(err).Warn("searching user by lf-email failed") - return + if _, ok := err.(*utils.UserNotFound); ok { + log.WithFields(f).Debug("unable to locate user by lf-email") + } else { + log.WithFields(f).WithError(err).Warn("searching user by lf-email failed") + return r + } } + // If found - just return if userModel != nil { - return + if !needToStoreUser { + return r + } + ctx := context.WithValue(r.Context(), "user", userModel) // nolint + return r.WithContext(ctx) } // Attempt to create the user newUser := &models.User{ - LfEmail: claUser.LFEmail, + LfEmail: strfmt.Email(claUser.LFEmail), LfUsername: claUser.LFUsername, Username: claUser.Name, } - log.WithFields(f).WithField("user", newUser).Debug("creating new user") + log.WithFields(f).Debug("creating new user") userModel, err = usersService.CreateUser(newUser, nil) if err != nil { log.WithFields(f).WithField("user", newUser).WithError(err).Warn("creating new user failed") - return + return r } // Log the event @@ -632,4 +725,9 @@ func createUserFromRequest(authorizer auth.Authorizer, usersService users.Servic UserModel: userModel, EventData: &events.UserCreatedEventData{}, }) + if !needToStoreUser { + return r + } + ctx := context.WithValue(r.Context(), "user", userModel) // nolint + return r.WithContext(ctx) } diff --git a/cla-backend-go/cmd/server_aws_lambda.go b/cla-backend-go/cmd/server_aws_lambda.go index 65a0dea80..a98fc76c9 100644 --- a/cla-backend-go/cmd/server_aws_lambda.go +++ b/cla-backend-go/cmd/server_aws_lambda.go @@ -1,3 +1,4 @@ +//go:build aws_lambda // +build aws_lambda // Copyright The Linux Foundation and each contributor to CommunityBridge. @@ -8,7 +9,7 @@ package cmd import ( "github.com/LF-Engineering/aws-lambda-go-api-proxy/httpadapter" "github.com/aws/aws-lambda-go/lambda" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/spf13/cobra" ) diff --git a/cla-backend-go/cmd/server_standalone.go b/cla-backend-go/cmd/server_standalone.go index fe2e193a1..d8f5b067a 100644 --- a/cla-backend-go/cmd/server_standalone.go +++ b/cla-backend-go/cmd/server_standalone.go @@ -1,3 +1,4 @@ +//go:build !aws_lambda // +build !aws_lambda // Copyright The Linux Foundation and each contributor to CommunityBridge. @@ -12,7 +13,7 @@ import ( "os/signal" "syscall" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -26,7 +27,7 @@ func runServer(cmd *cobra.Command, args []string) { errs := make(chan error, 2) go func() { log.Infof("Running http server on port: %d - set PORT environment variable to change port", viper.GetInt("PORT")) - errs <- http.ListenAndServe(fmt.Sprintf(":%d", viper.GetInt("PORT")), handler) + errs <- http.ListenAndServe(fmt.Sprintf(":%d", viper.GetInt("PORT")), handler) // nolint gosec no support for setting timeouts }() go func() { c := make(chan os.Signal) diff --git a/cla-backend-go/userSubscribeLambda/cmd/serve_lambda.go b/cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_lambda.go similarity index 95% rename from cla-backend-go/userSubscribeLambda/cmd/serve_lambda.go rename to cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_lambda.go index 31c44a848..e5a61ff21 100644 --- a/cla-backend-go/userSubscribeLambda/cmd/serve_lambda.go +++ b/cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_lambda.go @@ -1,3 +1,4 @@ +//go:build aws_lambda // +build aws_lambda // Copyright The Linux Foundation and each contributor to CommunityBridge. diff --git a/cla-backend-go/userSubscribeLambda/cmd/serve_local.go b/cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_local.go similarity index 90% rename from cla-backend-go/userSubscribeLambda/cmd/serve_local.go rename to cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_local.go index 8d15e4e64..220fe39f8 100644 --- a/cla-backend-go/userSubscribeLambda/cmd/serve_local.go +++ b/cla-backend-go/cmd/user-subscribe-lambda/cmd/serve_local.go @@ -1,3 +1,4 @@ +//go:build !aws_lambda // +build !aws_lambda // Copyright The Linux Foundation and each contributor to CommunityBridge. @@ -8,7 +9,7 @@ package cmd import ( "context" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -23,7 +24,7 @@ func postSQSEvent(w http.ResponseWriter, r *http.Request) { return } - dataByte, err := ioutil.ReadAll(r.Body) + dataByte, err := io.ReadAll(r.Body) if err != nil { log.Println("Failed to read body") http.Error(w, "Failed to read body", http.StatusInternalServerError) @@ -66,7 +67,7 @@ func Start(hf fn) error { http.HandleFunc("/", postSQSEvent) fmt.Printf("Starting server for testing HTTP POST...\n") - if err := http.ListenAndServe(":8080", nil); err != nil { + if err := http.ListenAndServe(":8080", nil); err != nil { // nolint gosec http no support for setting timeouts log.Fatal(err) } return nil diff --git a/cla-backend-go/cmd/user-subscribe-lambda/main.go b/cla-backend-go/cmd/user-subscribe-lambda/main.go new file mode 100644 index 000000000..744551c06 --- /dev/null +++ b/cla-backend-go/cmd/user-subscribe-lambda/main.go @@ -0,0 +1,350 @@ +// Copyright The Linux Foundation and each contributor to CommunityBridge. +// SPDX-License-Identifier: MIT + +package main + +import ( + "context" + "fmt" + "os" + "runtime" + + "github.com/linuxfoundation/easycla/cla-backend-go/cmd/user-subscribe-lambda/cmd" + + "github.com/go-openapi/strfmt" + + "github.com/linuxfoundation/easycla/cla-backend-go/utils" + "github.com/sirupsen/logrus" + + "github.com/LF-Engineering/lfx-models/models/event" + usersModels "github.com/LF-Engineering/lfx-models/models/users" + "github.com/aws/aws-lambda-go/events" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/linuxfoundation/easycla/cla-backend-go/config" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/token" + "github.com/linuxfoundation/easycla/cla-backend-go/users" + user_service "github.com/linuxfoundation/easycla/cla-backend-go/v2/user-service" + "github.com/mitchellh/mapstructure" +) + +// Build and version variables defined and set during the build process +var ( + // version the application version + version string + + // build/Commit the application build number + commit string + + // build date + buildDate string +) + +func init() { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.init", + } + var awsSession = session.Must(session.NewSession(&aws.Config{})) + stage := os.Getenv("STAGE") + if stage == "" { + log.WithFields(f).Fatal("stage not set") + } + log.WithFields(f).Infof("STAGE set to %s\n", stage) + configFile, err := config.LoadConfig("", awsSession, stage) + if err != nil { + log.WithFields(f).WithError(err).Panicf("Unable to load config - Error: %v", err) + } + + token.Init(configFile.Auth0Platform.ClientID, configFile.Auth0Platform.ClientSecret, configFile.Auth0Platform.URL, configFile.Auth0Platform.Audience) + user_service.InitClient(configFile.APIGatewayURL, configFile.AcsAPIKey) +} + +// Handler is the user subscribe handler lambda entry function +func Handler(ctx context.Context, snsEvent events.SNSEvent) error { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.Handler", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + } + if len(snsEvent.Records) == 0 { + log.WithFields(f).Warn("SNS event contained 0 records - ignoring message.") + return nil + } + + for _, message := range snsEvent.Records { + log.WithFields(f).Infof("Processing message id: '%s' for event source '%s'", message.SNS.MessageID, message.EventSource) + + log.WithFields(f).Debugf("Unmarshalling message body: '%s'", message.SNS.Message) + var model event.Event + err := model.UnmarshalBinary([]byte(message.SNS.Message)) + if err != nil { + log.WithFields(f).Warnf("Error: %v, JSON unmarshal failed - unable to process message: %s", err, message.SNS.MessageID) + return err + } + + f["modelType"] = model.Type + log.WithFields(f).Debugf("Processing message type: %s", model.Type) + switch model.Type { + case "UserSignedUp": + log.WithFields(f).Debugf("Detected message type: %s - processing...", model.Type) + Create(ctx, model) + case "UserUpdatedProfile": + log.WithFields(f).Debugf("Detected message type: %s - processing...", model.Type) + Update(ctx, model) + case "UserAuthenticated": + log.WithFields(f).Debugf("Ignoring message type: %s", model.Type) + default: + log.WithFields(f).Warnf("unrecognized message type: %s - unable to process message ", model.Type) + } + + } + return nil +} + +// Create saves the user data model to persistent storage +func Create(ctx context.Context, user event.Event) { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.Create", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + } + + uc := &usersModels.UserCreated{} + err := mapstructure.Decode(user.Data, uc) + if err != nil { + log.WithFields(f).WithError(err).Warn("unable to decode event") + return + } + + var userDetails *models.User + var userErr error + var awsSession = session.Must(session.NewSession(&aws.Config{})) + + stage := os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + usersRepo := users.NewRepository(awsSession, stage) + + log.WithFields(f).Debugf("locating user by username: %s in EasyCLA's database...", uc.Username) + userDetails, userErr = usersRepo.GetUserByLFUserName(uc.Username) + if userErr != nil { + log.WithFields(f).WithError(userErr).Warnf("unable to locate user by LfUsername: %s", uc.Username) + } + + if userDetails != nil { + log.WithFields(f).Warnf("unable to create user - user already created: %s", uc.Username) + } + + userServiceClient := user_service.GetClient() + log.WithFields(f).Debugf("locating user by username: %s in the user service...", uc.Username) + sfdcUserObject, err := userServiceClient.GetUserByUsername(uc.Username) + if err != nil { + log.WithFields(f).WithError(err).Warnf("unable to locate user by username: %s", uc.Username) + return + } + if sfdcUserObject == nil { + log.WithFields(f).Debugf("User-service model is nil so skipping user %s", uc.Username) + return + } + + log.WithFields(f).Debugf("Salesforce user-service object : %+v", sfdcUserObject) + + var primaryEmail string + var emails []string + for _, email := range sfdcUserObject.Emails { + if *email.IsPrimary { + primaryEmail = *email.EmailAddress + } + emails = append(emails, *email.EmailAddress) + } + + _, nowStr := utils.CurrentTime() + createUserModel := &models.User{ + Admin: false, + DateCreated: nowStr, + DateModified: nowStr, + Emails: emails, + LfEmail: strfmt.Email(primaryEmail), + LfUsername: sfdcUserObject.Username, + Note: "Create via user-service event", + UserExternalID: sfdcUserObject.ID, + UserID: userDetails.UserID, + Username: fmt.Sprintf("%s %s", sfdcUserObject.FirstName, sfdcUserObject.LastName), + Version: "v1", + } + + log.WithFields(f).Debugf("Creating user in Dynamo DB : %+v", createUserModel) + _, createErr := usersRepo.CreateUser(createUserModel) + if createErr != nil { + log.WithFields(f).Warnf("unable to create user by LfUsername: %s", uc.Username) + return + } +} + +// Update saves the user data model to persistent storage +func Update(ctx context.Context, user event.Event) { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.Update", + utils.XREQUESTID: ctx.Value(utils.XREQUESTID), + } + + uc := &usersModels.UserUpdated{} + err := mapstructure.Decode(user.Data, uc) + if err != nil { + log.WithFields(f).WithError(err).Warn("unable to decode event") + return + } + + var userDetails *models.User + var userErr error + var awsSession = session.Must(session.NewSession(&aws.Config{})) + + stage := os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + usersRepo := users.NewRepository(awsSession, stage) + + userDetails, userErr = usersRepo.GetUserByLFUserName(*uc.Username) + if userErr != nil { + log.WithFields(f).WithError(userErr).Warnf("unable to locate user by LfUsername: %s", *uc.Username) + } + + if userDetails == nil { + for _, email := range uc.Emails { + userDetails, userErr = usersRepo.GetUserByEmail(*email.EmailAddress) + if userErr != nil { + log.WithFields(f).WithError(userErr).Warnf("unable to locate user by LfUsername: %s", *uc.Username) + } + } + } + + if userDetails == nil { + userDetails, userErr = usersRepo.GetUserByExternalID(uc.UserID) + if userErr != nil { + log.WithFields(f).WithError(userErr).Warnf("unable to locate user by UserExternalID: %s", uc.UserID) + } + } + + if userDetails == nil { + log.WithFields(f).Debugf("User model is nil - adding as new user %s...", *uc.Username) + // Attempt to create the user from the upate model + createFromUpdateErr := createUserFromUpdatedModel(uc) + if createFromUpdateErr != nil { + log.WithFields(f).WithError(createFromUpdateErr).Warnf("unable to create new user record from user service update message: %s", uc.UserID) + } + return + } + + userServiceClient := user_service.GetClient() + sfdcUserObject, err := userServiceClient.GetUser(uc.UserID) + if err != nil { + log.WithFields(f).WithError(err).Warnf("unable to locate user by SFID: %s, error: %+v", uc.UserID, userErr) + return + } + + log.WithFields(f).Debugf("Salesforce user-service object : %+v", sfdcUserObject) + + if sfdcUserObject == nil { + log.WithFields(f).Debugf("User-service model is nil so skipping user %s with SFID %s", *uc.Username, uc.UserID) + return + } + + var primaryEmail string + var emails []string + for _, email := range sfdcUserObject.Emails { + if *email.IsPrimary { + primaryEmail = *email.EmailAddress + } + emails = append(emails, *email.EmailAddress) + } + + updateUserModel := &models.UserUpdate{ + Emails: emails, + LfEmail: primaryEmail, + LfUsername: sfdcUserObject.Username, + Note: "Update via user-service event", + UserExternalID: sfdcUserObject.ID, + UserID: userDetails.UserID, + Username: fmt.Sprintf("%s %s", sfdcUserObject.FirstName, sfdcUserObject.LastName), + } + + log.WithFields(f).Debugf("Updating user in Dynamo DB : %+v", updateUserModel) + _, updateErr := usersRepo.Save(updateUserModel) + if updateErr != nil { + log.WithFields(f).Warnf("Error - unable to update user by LfUsername: %s, error: %+v", *uc.Username, updateErr) + return + } +} + +func createUserFromUpdatedModel(userModelUpdated *usersModels.UserUpdated) error { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.createUserFromUpdatedModel", + "userID": userModelUpdated.UserID, + "userName": userModelUpdated.Username, + } + + var awsSession = session.Must(session.NewSession(&aws.Config{})) + + stage := os.Getenv("STAGE") + if stage == "" { + log.Fatal("stage not set") + } + userServiceClient := user_service.GetClient() + sfdcUserObject, err := userServiceClient.GetUser(userModelUpdated.UserID) + if err != nil { + log.WithFields(f).WithError(err).Warnf("unable to locate user by ID: %s", userModelUpdated.UserID) + return err + } + + var primaryEmail string + var emails []string + for _, email := range sfdcUserObject.Emails { + if *email.IsPrimary { + primaryEmail = *email.EmailAddress + } + emails = append(emails, *email.EmailAddress) + } + + newUserModel := &models.User{ + Emails: emails, + LfEmail: strfmt.Email(primaryEmail), + LfUsername: sfdcUserObject.Username, + Note: "Update via user-service event", + UserExternalID: sfdcUserObject.ID, + UserID: userModelUpdated.UserID, + Username: fmt.Sprintf("%s %s", sfdcUserObject.FirstName, sfdcUserObject.LastName), + } + + log.WithFields(f).Debugf("Creating user in Dynamo DB : %+v", newUserModel) + usersRepo := users.NewRepository(awsSession, stage) + + _, createErr := usersRepo.CreateUser(newUserModel) + if createErr != nil { + log.WithFields(f).WithError(createErr).Warnf("unable to create user by LfUsername: %s", *userModelUpdated.Username) + return createErr + } + + return nil +} + +func main() { + f := logrus.Fields{ + "functionName": "userSubscribeLambda.main.main", + } + var err error + + // Show the version and build info + log.WithFields(f).Infof("Name : userSubscribe handler") + log.WithFields(f).Infof("Version : %s", version) + log.WithFields(f).Infof("Git commit hash : %s", commit) + log.WithFields(f).Infof("Build date : %s", buildDate) + log.WithFields(f).Infof("Golang OS : %s", runtime.GOOS) + log.WithFields(f).Infof("Golang Arch : %s", runtime.GOARCH) + + err = cmd.Start(Handler) + if err != nil { + log.WithFields(f).WithError(err).Fatal(err) + } +} diff --git a/cla-backend-go/cmd/zipbuilder_lambda/main.go b/cla-backend-go/cmd/zipbuilder_lambda/main.go index cb0bc1375..b8296864a 100644 --- a/cla-backend-go/cmd/zipbuilder_lambda/main.go +++ b/cla-backend-go/cmd/zipbuilder_lambda/main.go @@ -7,15 +7,15 @@ import ( "context" "os" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" - "github.com/communitybridge/easycla/cla-backend-go/v2/signatures" + "github.com/linuxfoundation/easycla/cla-backend-go/v2/signatures" "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" ) var ( @@ -36,6 +36,7 @@ var ( type BuildZipEvent struct { ClaGroupID string `json:"cla_group_id"` SignatureType string `json:"signature_type"` + FileType string `json:"file_type"` } var zipBuilder signatures.ZipBuilder @@ -59,12 +60,30 @@ func handler(ctx context.Context, event BuildZipEvent) error { var err error log.WithField("event", event).Debug("zip builder called") switch event.SignatureType { - case signatures.ICLA: - err = zipBuilder.BuildICLAZip(event.ClaGroupID) - case signatures.CCLA: - err = zipBuilder.BuildCCLAZip(event.ClaGroupID) + case utils.ClaTypeICLA: + if event.FileType == utils.FileTypePDF { + err = zipBuilder.BuildICLAPDFZip(event.ClaGroupID) + } else if event.FileType == utils.FileTypeCSV { + err = zipBuilder.BuildICLACSVZip(event.ClaGroupID) + } else { + log.WithField("event", event).Warn("Invalid event") + } + case utils.ClaTypeCCLA: + if event.FileType == utils.FileTypePDF { + err = zipBuilder.BuildCCLAPDFZip(event.ClaGroupID) + } else if event.FileType == utils.FileTypeCSV { + err = zipBuilder.BuildCCLACSVZip(event.ClaGroupID) + } else { + log.WithField("event", event).Warn("Invalid event") + } + case utils.ClaTypeECLA: + if event.FileType == utils.FileTypeCSV { + err = zipBuilder.BuildECLACSVZip(event.ClaGroupID) + } else { + log.WithField("event", event).Warn("Invalid event") + } default: - log.WithField("event", event).Debug("Invalid event") + log.WithField("event", event).Warn("Invalid event") } if err != nil { log.WithField("args", event).Error("failed to build zip", err) @@ -83,10 +102,10 @@ func main() { log.Info("Lambda server starting...") printBuildInfo() if os.Getenv("LOCAL_MODE") == "true" { - if len(os.Args) != 3 { - log.Fatal("invalid number of args. first arg should be icla or ccla and 2nd arg should be cla_group_id") + if len(os.Args) != 4 { + log.Fatal("invalid number of args. first arg should be icla or ccla, 2nd should be pdf or csv and 3rd arg should be cla_group_id") } - err := handler(utils.NewContext(), BuildZipEvent{SignatureType: os.Args[1], ClaGroupID: os.Args[2]}) + err := handler(utils.NewContext(), BuildZipEvent{SignatureType: os.Args[1], FileType: os.Args[2], ClaGroupID: os.Args[3]}) if err != nil { log.Fatal(err) } diff --git a/cla-backend-go/cmd/zipbuilder_scheduler_lambda/main.go b/cla-backend-go/cmd/zipbuilder_scheduler_lambda/main.go index c15063889..eb8fc8c81 100644 --- a/cla-backend-go/cmd/zipbuilder_scheduler_lambda/main.go +++ b/cla-backend-go/cmd/zipbuilder_scheduler_lambda/main.go @@ -10,7 +10,7 @@ import ( "os" "sync" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute" "github.com/aws/aws-sdk-go/service/dynamodb/expression" @@ -24,7 +24,7 @@ import ( awslambda "github.com/aws/aws-lambda-go/lambda" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" - log "github.com/communitybridge/easycla/cla-backend-go/logging" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" ) var ( @@ -52,6 +52,7 @@ type ClaGroup struct { type BuildZipEvent struct { ClaGroupID string `json:"cla_group_id"` SignatureType string `json:"signature_type"` + FileType string `json:"file_type"` } func handler(ctx context.Context, event events.CloudWatchEvent) { @@ -71,13 +72,30 @@ func handler(ctx context.Context, event events.CloudWatchEvent) { if claGroup.ProjectCclaEnabled { eventPayloads = append(eventPayloads, BuildZipEvent{ ClaGroupID: claGroup.ProjectID, - SignatureType: "ccla", + SignatureType: utils.ClaTypeCCLA, + FileType: utils.FileTypePDF, + }) + eventPayloads = append(eventPayloads, BuildZipEvent{ + ClaGroupID: claGroup.ProjectID, + SignatureType: utils.ClaTypeCCLA, + FileType: utils.FileTypeCSV, + }) + eventPayloads = append(eventPayloads, BuildZipEvent{ + ClaGroupID: claGroup.ProjectID, + SignatureType: utils.ClaTypeECLA, + FileType: utils.FileTypeCSV, }) } if claGroup.ProjectIclaEnabled { eventPayloads = append(eventPayloads, BuildZipEvent{ ClaGroupID: claGroup.ProjectID, - SignatureType: "icla", + SignatureType: utils.ClaTypeICLA, + FileType: utils.FileTypePDF, + }) + eventPayloads = append(eventPayloads, BuildZipEvent{ + ClaGroupID: claGroup.ProjectID, + SignatureType: utils.ClaTypeICLA, + FileType: utils.FileTypeCSV, }) } } @@ -96,13 +114,13 @@ func handler(ctx context.Context, event events.CloudWatchEvent) { func invokeLambda(wg *sync.WaitGroup, lambdaClient *lambda.Lambda, stage string, buildZipEvent BuildZipEvent) { defer wg.Done() - log.WithField("buildZipEvent", buildZipEvent).Debug("invoking zipbuilder-lambda") + log.WithField("buildZipEvent", buildZipEvent).Debug("invoking zip-builder-lambda") payload, err := json.Marshal(buildZipEvent) if err != nil { log.Error("Error marshalling BuildZip request", err) return } - functionName := fmt.Sprintf("cla-backend-%s-zipbuilder-lambda", stage) + functionName := fmt.Sprintf("cla-backend-%s-zip-builder-lambda", stage) _, err = lambdaClient.Invoke(&lambda.InvokeInput{FunctionName: aws.String(functionName), Payload: payload}) if err != nil { diff --git a/cla-backend-go/company/handlers.go b/cla-backend-go/company/handlers.go index 1085b5ed6..4312d9219 100644 --- a/cla-backend-go/company/handlers.go +++ b/cla-backend-go/company/handlers.go @@ -10,19 +10,19 @@ import ( "github.com/sirupsen/logrus" - "github.com/communitybridge/easycla/cla-backend-go/utils" + "github.com/linuxfoundation/easycla/cla-backend-go/utils" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations/organization" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/organization" - "github.com/communitybridge/easycla/cla-backend-go/events" - "github.com/communitybridge/easycla/cla-backend-go/users" + "github.com/linuxfoundation/easycla/cla-backend-go/events" + "github.com/linuxfoundation/easycla/cla-backend-go/users" - "github.com/communitybridge/easycla/cla-backend-go/gen/models" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations" - "github.com/communitybridge/easycla/cla-backend-go/gen/restapi/operations/company" - log "github.com/communitybridge/easycla/cla-backend-go/logging" - "github.com/communitybridge/easycla/cla-backend-go/user" - orgService "github.com/communitybridge/easycla/cla-backend-go/v2/organization-service" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/models" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations" + "github.com/linuxfoundation/easycla/cla-backend-go/gen/v1/restapi/operations/company" + log "github.com/linuxfoundation/easycla/cla-backend-go/logging" + "github.com/linuxfoundation/easycla/cla-backend-go/user" + orgService "github.com/linuxfoundation/easycla/cla-backend-go/v2/organization-service" "github.com/go-openapi/runtime/middleware" ) @@ -36,7 +36,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companiesModel, err := service.GetCompanies(ctx) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to query all companies, error: %v", err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompaniesBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -52,7 +52,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companyModel, err := service.GetCompany(ctx, params.CompanyID) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to query company by ID: %s, error: %v", params.CompanyID, err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompanyBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -87,7 +87,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companyModel, err := service.GetCompanyByExternalID(ctx, params.CompanySFID) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to get associated salesforce Organization: %s using SFID: %s, error: %v", org.Name, params.CompanySFID, err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompanyByExternalIDBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -108,7 +108,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companyModel, err := service.GetCompanyBySigningEntityName(ctx, params.Name, params.CompanySFID) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - Unable to locate Company with Signing Entity Request of %s", params.Name) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompanyBySigningEntityNameBadRequest().WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -134,7 +134,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companiesModel, err := service.SearchCompanyByName(ctx, params.CompanyName, nextKey) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to query company by name: %s, error: %v", params.CompanyName, err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewSearchCompanyBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -173,7 +173,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companies, err := service.GetCompaniesByUserManager(ctx, params.UserID) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to query companies by user manager id: %s, error: %v", params.UserID, err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompaniesByUserManagerBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -207,7 +207,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv companies, err := service.GetCompaniesByUserManagerWithInvites(ctx, params.UserID) if err != nil { msg := fmt.Sprintf("EasyCLA - 400 Bad Request - unable to query companies by user manager id: %s, error: %v", params.UserID, err) - log.Warnf(msg) + log.Warnf("%s", msg) return company.NewGetCompaniesByUserManagerWithInvitesBadRequest().WithXRequestID(reqID).WithPayload(&models.ErrorResponse{ Code: "400", Message: msg, @@ -257,7 +257,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv return company.NewAddUsertoCompanyAccessListBadRequest().WithXRequestID(reqID) } - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.CompanyACLUserAdded, CompanyID: params.CompanyID, UserID: claUser.UserID, @@ -280,7 +280,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv } // Add an event to the log - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.CompanyACLRequestAdded, CompanyID: params.CompanyID, UserID: claUser.UserID, @@ -305,7 +305,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv } // Add an event to the log - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.CompanyACLRequestApproved, CompanyID: params.CompanyID, UserID: claUser.UserID, @@ -330,7 +330,7 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv } // Add an event to the log - eventsService.LogEvent(&events.LogEventArgs{ + eventsService.LogEventWithContext(ctx, &events.LogEventArgs{ EventType: events.CompanyACLRequestDenied, CompanyID: params.CompanyID, UserID: claUser.UserID, @@ -347,17 +347,27 @@ func Configure(api *operations.ClaAPI, service IService, usersService users.Serv api.OrganizationSearchOrganizationHandler = organization.SearchOrganizationHandlerFunc(func(params organization.SearchOrganizationParams) middleware.Responder { reqID := utils.GetRequestID(params.XREQUESTID) ctx := context.WithValue(context.Background(), utils.XREQUESTID, reqID) // nolint + f := logrus.Fields{ + "functionName": "company.handler.OrganizationSearchOrganizationHandler", + "companyName": params.CompanyName, + "websiteName": params.WebsiteName, + "includeSigningEntityName": params.IncludeSigningEntityName, + } if params.CompanyName == nil && params.WebsiteName == nil && params.DollarFilter == nil { - log.Debugf("CompanyName or WebsiteName or filter atleast one required") + log.WithFields(f).Debugf("CompanyName or WebsiteName or filter at least one required") return organization.NewSearchOrganizationBadRequest().WithXRequestID(reqID).WithPayload(errorResponse(errors.New("companyName or websiteName or filter at least one required"))) } companyName, websiteName, filter := validateParams(params) - result, err := service.SearchOrganizationByName(ctx, companyName, websiteName, filter) + result, err := service.SearchOrganizationByName(ctx, companyName, websiteName, utils.BoolValue(params.IncludeSigningEntityName), filter) if err != nil { - log.Warnf("error occured while search org %s. error = %s", *params.CompanyName, err.Error()) + companyName := "