EVA-4114 - Automated deployment #98
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,189 @@ | ||
|
|
||
| stages: | ||
| - build | ||
| - deploy | ||
|
|
||
| variables: | ||
| DOCKER_REGISTRY: dockerhub.ebi.ac.uk | ||
| DOCKER_IMAGE: dockerhub.ebi.ac.uk/ebivariation/eva-seqcol | ||
| K8S_REPO_URL: https://github.com/EBIvariation/eva-k8s.git | ||
|
|
||
| # REQUIRED GitLab CI/CD Variables (Global): | ||
| # DOCKER_REGISTRY_USER: Username for Docker registry authentication | ||
| # DOCKER_REGISTRY_PASSWORD: Password for Docker registry authentication | ||
| # K8S_REPO_DEPLOY_KEY: Base64-encoded SSH private key for cloning/pushing to eva-k8s GitHub repo | ||
| # | ||
| # DEVELOPMENT ENVIRONMENT VARIABLES (suffixed with _DEV): | ||
| # METADATA_DB_HOST_DEV: PostgreSQL host | ||
| # METADATA_DB_PORT_DEV: PostgreSQL port | ||
| # METADATA_DB_USER_DEV: PostgreSQL username | ||
| # METADATA_DB_PASSWORD_DEV: PostgreSQL password | ||
| # METADATA_DB_SPRING_DATASOURCE_URL_DEV: JDBC connection string | ||
| # SEQCOL_ADMIN_USER_DEV: Admin username | ||
| # SEQCOL_ADMIN_PASSWORD_DEV: Admin password | ||
| # KUBECONFIG_CONTENT_DEV: Base64-encoded kubeconfig for dev cluster | ||
| # | ||
| # PRODUCTION ENVIRONMENT VARIABLES (suffixed with _PROD): | ||
| # METADATA_DB_HOST_PROD: PostgreSQL host | ||
| # METADATA_DB_PORT_PROD: PostgreSQL port | ||
| # METADATA_DB_USER_PROD: PostgreSQL username | ||
| # METADATA_DB_PASSWORD_PROD: PostgreSQL password | ||
| # METADATA_DB_SPRING_DATASOURCE_URL_PROD: JDBC connection string | ||
| # SEQCOL_ADMIN_USER_PROD: Admin username | ||
| # SEQCOL_ADMIN_PASSWORD_PROD: Admin password | ||
| # KUBECONFIG_CONTENT_PROD: Base64-encoded kubeconfig for prod cluster | ||
|
|
||
| # ============================================================================ | ||
| # Base Job Templates | ||
| # ============================================================================ | ||
|
|
||
| .build_docker: | ||
| stage: build | ||
| image: docker:latest | ||
| services: | ||
| - docker:dind | ||
| variables: | ||
| DOCKER_DRIVER: overlay2 | ||
| DOCKER_TLS_CERTDIR: "/certs" | ||
| before_script: | ||
| - echo $DOCKER_REGISTRY_PASSWORD | docker login -u $DOCKER_REGISTRY_USER --password-stdin $DOCKER_REGISTRY | ||
| script: | ||
| - docker build -t $DOCKER_IMAGE:$IMAGE_TAG -t $DOCKER_IMAGE:$IMAGE_LATEST . | ||
| - docker push $DOCKER_IMAGE:$IMAGE_TAG | ||
| - docker push $DOCKER_IMAGE:$IMAGE_LATEST | ||
| - echo "IMAGE_TAG=$IMAGE_TAG" > build.env | ||
| artifacts: | ||
| reports: | ||
| dotenv: build.env | ||
|
|
||
| .update_in_github: | ||
| stage: deploy | ||
| image: alpine/git:latest | ||
| before_script: | ||
| - apk add --no-cache bash curl | ||
| - mkdir -p ~/.ssh | ||
| - echo "$K8S_REPO_DEPLOY_KEY" | base64 -d > ~/.ssh/id_rsa | ||
| - chmod 600 ~/.ssh/id_rsa | ||
| - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts 2>/dev/null | ||
| - git config --global user.email "gitlab-ci@ebi.ac.uk" | ||
| - git config --global user.name "GitLab CI" | ||
| script: | ||
| - git clone --depth 1 git@github.com:EBIvariation/eva-k8s.git k8s-repo | ||
| - cd k8s-repo | ||
| - | | ||
| sed -i "s/newTag:.*/newTag: $IMAGE_TAG/" $K8S_MANIFEST_PATH/kustomization.yaml | ||
| grep "newTag:" $K8S_MANIFEST_PATH/kustomization.yaml | ||
| - git add $K8S_MANIFEST_PATH/kustomization.yaml | ||
| - git commit -m "chore: update eva-seqcol $ENV image tag to $IMAGE_TAG" | ||
| - git push origin main | ||
|
|
||
| .deploy_to_cluster: | ||
| stage: deploy | ||
| image: alpine/k8s:latest | ||
| before_script: | ||
| - mkdir -p ~/.kube | ||
| - echo "$KUBECONFIG_CONTENT" | base64 -d > ~/.kube/config | ||
| - chmod 600 ~/.kube/config | ||
| # Generate secrets from CI variables into manifest path | ||
| - mkdir -p $K8S_MANIFEST_PATH | ||
| - printf "host=%s\nport=%s\nusername=%s\npassword=%s\n" "$DB_HOST" "$DB_PORT" "$DB_USER" "$DB_PASSWORD" > $K8S_MANIFEST_PATH/secrets-db.env | ||
| - printf "username=%s\npassword=%s\n" "$ADMIN_USER" "$ADMIN_PASSWORD" > $K8S_MANIFEST_PATH/secrets-admin.env | ||
| - printf "SPRING_DATASOURCE_URL=%s\nDDL_BEHAVIOUR=update\nSPRING_JPA_PROPERTIES_HIBERNATE_DEFAULT_SCHEMA=seqcol\nFTP_PROXY_HOST=%s\nFTP_PROXY_PORT=%s\nSCAFFOLDS_ENABLED=%s\n" "$SPRING_DATASOURCE_URL" "${FTP_PROXY_HOST:-}" "${FTP_PROXY_PORT:-0}" "${SCAFFOLDS_ENABLED:-true}" > $K8S_MANIFEST_PATH/config.env | ||
| script: | ||
| - kubectl apply -k $K8S_MANIFEST_PATH | ||
| - kubectl rollout status deployment/eva-seqcol -n $K8S_NAMESPACE --timeout=5m | ||
| environment: | ||
| kubernetes: | ||
| namespace: $K8S_NAMESPACE | ||
|
|
||
| # ============================================================================ | ||
| # PRODUCTION: Deploy on git tags | ||
| # ============================================================================ | ||
|
|
||
| build:docker:prod: | ||
| extends: .build_docker | ||
| variables: | ||
| IMAGE_TAG: $CI_COMMIT_TAG | ||
| IMAGE_LATEST: latest | ||
| only: | ||
| - tags | ||
|
|
||
| update-in-github:prod: | ||
| extends: .update_in_github | ||
| needs: | ||
| - build:docker:prod | ||
| variables: | ||
| ENV: prod | ||
| K8S_MANIFEST_PATH: k8s-manifests/eva-seqcol/overlays/prod | ||
| environment: | ||
| name: prod | ||
| only: | ||
| - tags | ||
|
|
||
| deploy-to-cluster:prod: | ||
| extends: .deploy_to_cluster | ||
| needs: | ||
| - update-in-github:prod | ||
| variables: | ||
| K8S_MANIFEST_PATH: k8s-manifests/eva-seqcol/overlays/prod | ||
| K8S_NAMESPACE: eva-seqcol-prod | ||
| # Production-specific environment variables | ||
| DB_HOST: $METADATA_DB_HOST_PROD | ||
| DB_PORT: $METADATA_DB_PORT_PROD | ||
| DB_USER: $METADATA_DB_USER_PROD | ||
| DB_PASSWORD: $METADATA_DB_PASSWORD_PROD | ||
| SPRING_DATASOURCE_URL: $METADATA_DB_SPRING_DATASOURCE_URL_PROD | ||
| ADMIN_USER: $SEQCOL_ADMIN_USER_PROD | ||
| ADMIN_PASSWORD: $SEQCOL_ADMIN_PASSWORD_PROD | ||
| KUBECONFIG_CONTENT: $KUBECONFIG_CONTENT_PROD | ||
| environment: | ||
| name: prod | ||
| only: | ||
| - tags | ||
|
|
||
| # ============================================================================ | ||
| # DEVELOPMENT: Deploy on main branch (manual trigger) | ||
|
Contributor
There was a problem hiding this comment. What's the reasoning behind making the dev deploys manual, but the prod ones automatic?
Member
Author
There was a problem hiding this comment. The initial rational was to allow more deployments on dev prior to merging while not deploying for every push. However After looking into it I realised that I could not deploy any code from forks because they are not mirrored. I'll remove the manual trigger. |
||
| # ============================================================================ | ||
|
|
||
| build:docker:dev: | ||
| extends: .build_docker | ||
| variables: | ||
| IMAGE_TAG: dev-$CI_COMMIT_SHORT_SHA | ||
| IMAGE_LATEST: dev-latest | ||
| only: | ||
| - main | ||
|
|
||
| update-in-github:dev: | ||
| extends: .update_in_github | ||
| needs: | ||
| - build:docker:dev | ||
| variables: | ||
| ENV: dev | ||
| K8S_MANIFEST_PATH: k8s-manifests/eva-seqcol/overlays/dev | ||
| environment: | ||
| name: dev | ||
| when: manual | ||
| only: | ||
| - main | ||
|
|
||
| deploy-to-cluster:dev: | ||
| extends: .deploy_to_cluster | ||
| needs: | ||
| - update-in-github:dev | ||
| variables: | ||
| K8S_MANIFEST_PATH: k8s-manifests/eva-seqcol/overlays/dev | ||
| K8S_NAMESPACE: eva-seqcol-dev | ||
| # Development-specific environment variables | ||
| DB_HOST: $METADATA_DB_HOST_DEV | ||
| DB_PORT: $METADATA_DB_PORT_DEV | ||
| DB_USER: $METADATA_DB_USER_DEV | ||
| DB_PASSWORD: $METADATA_DB_PASSWORD_DEV | ||
| SPRING_DATASOURCE_URL: $METADATA_DB_SPRING_DATASOURCE_URL_DEV | ||
| ADMIN_USER: $SEQCOL_ADMIN_USER_DEV | ||
| ADMIN_PASSWORD: $SEQCOL_ADMIN_PASSWORD_DEV | ||
| KUBECONFIG_CONTENT: $KUBECONFIG_CONTENT_DEV | ||
| environment: | ||
| name: dev | ||
| when: manual | ||
| only: | ||
| - main | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,30 +1,31 @@ | ||
|
|
||
|
Contributor
There was a problem hiding this comment. Is there a way to use a single environment variable to select the profile within The approach here is probably fine, I'm just not sure I like the idea of storing so many properties in the Gitlab CI, it seems less legible and harder to maintain.
Member
Author
There was a problem hiding this comment. I agree that this is not optimal and that the number of env variable in Gitlab will quickly get out of hands
We could also look into how we can manage env variable outside of gitlab variables |
||
| spring.profiles.active=seqcol | ||
| management.endpoints.web.exposure.include=info,health | ||
| management.endpoints.web.base-path=/ | ||
| management.info.git.mode=full | ||
|
|
||
| logging.level.uk.ac.ebi.eva.contigalias=DEBUG | ||
|
|
||
| # Admin credentials - override via environment variables | ||
| controller.auth.admin.username=${ADMIN_USER:admin} | ||
| controller.auth.admin.password=${ADMIN_PASSWORD:admin} | ||
|
|
||
| # Database configuration - override via environment variables | ||
| spring.datasource.url=${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/seqcol_db} | ||
| spring.datasource.username=${SPRING_DATASOURCE_USERNAME:postgres} | ||
| spring.datasource.password=${SPRING_DATASOURCE_PASSWORD:password} | ||
| spring.jpa.hibernate.ddl-auto=${SPRING_JPA_HIBERNATE_DDL_AUTO:update} | ||
| spring.datasource.driver-class-name=org.postgresql.Driver | ||
| spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect | ||
| spring.jpa.generate-ddl=true | ||
|
|
||
| server.servlet.context-path=/eva/webservices/seqcol | ||
| server.port=8081 | ||
|
|
||
| ftp.proxy.host=${FTP_PROXY_HOST:} | ||
| ftp.proxy.port=${FTP_PROXY_PORT:0} | ||
|
|
||
| config.scaffolds.enabled=${SCAFFOLDS_ENABLED:true} | ||
|
|
||
| asm.file.download.dir=/tmp | ||
| service.info.file.path=src/main/resources/static/service-info.json | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.