diff --git a/Makefile b/Makefile index 8a59b25..66ce3d6 100644 --- a/Makefile +++ b/Makefile @@ -1,67 +1,21 @@ SHELL := /bin/bash include .env +include includes/python.mk +include includes/terraform.mk PROJECT_ID=$(PROJECT)-$(ENV) GOOGLE_CLOUD_PROJECT=$(PROJECT_ID) .EXPORT_ALL_VARIABLES: +.DEFAULT_GOAL := help + .PHONY: all test check quality run docker # -- bucket definitions DEPLOY_BUCKET := $(PROJECT_ID)-gcs-deploy -check: poetry-lock clean-code clean-coverage quality test poetry-build docker-build -test: prepare-test poetry-test - -quality: prepare-quality poetry-quality -run: prepare-run poetry-run -docker: docker-build docker-run - -prepare-quality: - @poetry install --only nox,fmt,lint,type_check,docs - -poetry-quality: - @poetry run nox -s fmt_check - @poetry run nox -s lint - @poetry run nox -s type_check - @poetry run nox -s docs - -poetry-lock: - @poetry env use python3.9 - @poetry install - @poetry lock --check - -clean-coverage: - @rm -f .coverage* - @rm -f coverage.xml - -prepare-test: - @poetry install --only nox - -poetry-test: - @poetry run nox -s test-3.9 - @rm -f .coverage* - @rm -f coverage.xml - @poetry run nox -s test-3.10 - -clean-code: - @poetry run isort . - @poetry run black . - -poetry-build: - @poetry build - -prepare-run: - @poetry install --only main - -poetry-run: - @poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload - -poetry-pulumi: - @poetry run python -m iac-pulumi.main - docker-build: @docker build --tag dbt-serverless . @@ -87,6 +41,9 @@ docker-run: # This target will perform the complete setup of the current repository. # ---------------------------------------------------------------------------------------- # +help: ## Displays the current message + @awk -F ':.*?##' '/^[^\t].+?:.*?.*?##/ {printf "\033[36m%-30s\033[0m %s\n", $$1, $$NF}' $(MAKEFILE_LIST) + all: create-project create-bucket create-artifactregistry build deploy-app build: check build-app @@ -157,117 +114,3 @@ deploy-app: @echo "[$@] :: Pushing docker image" @docker push $(REGION)-docker.pkg.dev/$(PROJECT_ID)/$(REPOSITORY_ID)/dbt-serverless:latest; @echo "[$@] :: docker push is over." - -# ---------------------------------------------------------------------------------------- # -# -- < IaC > -- -# ---------------------------------------------------------------------------------------- # -# -- terraform variables declaration -IAC_DIR = iac/ -DBT_DIR = $(DBT_PROJECT)/ -TF_DIR = $(IAC_DIR).terraform/ -TF_INIT = $(TF_DIR)terraform.tfstate -TF_VARS = $(IAC_DIR)terraform.tfvars -TF_PLAN = $(IAC_DIR)tfplan -TF_STATE = $(wildcard $(IAC_DIR)*.tfstate $(TF_DIR)*.tfstate) -TF_FILES = $(wildcard $(IAC_DIR)*.tf) - -# -- this target will clean the local terraform infrastructure -.PHONY: iac-clean -iac-clean: - @echo "[$@] :: cleaning the infrastructure intermediary files" - @rm -fr $(TF_PLAN) $(TF_VARS); - @if [ ! -f $(IAC_DIR).iac-env ] || [ $$(cat $(IAC_DIR).iac-env || echo -n) != $(PROJECT_ID) ]; then \ - echo "[$@] :: env has changed, removing also $(TF_DIR) and $(IAC_DIR).terraform.lock.hcl"; \ - rm -rf $(TF_DIR) $(IAC_DIR).terraform.lock.hcl; \ - fi; - - @echo "[$@] :: infrastructure cleaning DONE" - -# -- this target will initialize the terraform initialization -.PHONY: iac-init -iac-init: $(TF_INIT) # provided for convenience -$(TF_INIT): - @set -euo pipefail; \ - if [ ! -d $(TF_DIR) ]; then \ - function remove_me() { if (( $$? != 0 )); then rm -fr $(TF_DIR); fi; }; \ - trap remove_me EXIT; \ - echo "[iac-init] :: initializing terraform"; \ - echo "$(PROJECT_ID)" > $(IAC_DIR).iac-env; \ - cd $(IAC_DIR) && terraform init \ - -backend-config=bucket=$(DEPLOY_BUCKET) \ - -backend-config=prefix=terraform-state/$(ENV) \ - -input=false; \ - else \ - echo "[iac-init] :: terraform already initialized"; \ - fi; - -# -- internal definition for easing changes -define HERE_TF_VARS -project = "$(PROJECT_ID)" -zone = "$(ZONE)" -region = "$(REGION)" -env = "$(ENV)" -repository_id = "$(REPOSITORY_ID)" -github_owner = "$(GITHUB_OWNER)" -github_repo = "$(GITHUB_REPO)" -github_token = "$(GITHUB_TOKEN)" -pypi_token = "$(PYPI_TOKEN)" -codecov_token = "$(CODECOV_TOKEN)" -endef -export HERE_TF_VARS - -# -- this target will create the terraform.tfvars file -.PHONY: iac-prepare -iac-prepare: $(TF_VARS) # provided for convenience -$(TF_VARS): $(TF_INIT) - @echo "[iac-prepare] :: generation of $(TF_VARS) file"; - @echo "$$HERE_TF_VARS" > $(TF_VARS); - @echo "[iac-prepare] :: generation of $(TF_VARS) file DONE."; - -# -- this target will create the tfplan file whenever the variables file and any *.tf -# file have changed -.PHONY: iac-plan iac-plan-clean -iac-plan-clean: - @rm -f $(TF_PLAN) - -iac-plan: iac-plan-clean $(TF_PLAN) # provided for convenience -$(TF_PLAN): $(TF_VARS) $(TF_FILES) - @echo "[iac-plan] :: planning the iac in $(PROJECT_ID)"; - @set -euo pipefail; \ - cd $(IAC_DIR) && terraform plan \ - -var-file $(shell basename $(TF_VARS)) \ - -out=$(shell basename $(TF_PLAN)); - @echo "[iac-plan] :: planning the iac for $(PROJECT_ID) DONE."; - -# -- this target will only trigger the iac of the current parent -.PHONY: iac-validate -iac-validate: - @echo "[$@] :: validating the infrastructure for $(PROJECT_ID)" - @set -euo pipefail; \ - cd $(IAC_DIR) && terraform validate; - @echo "[$@] :: infrastructure validated on $(PROJECT_ID)" - -# -- this target will only trigger the iac of the current parent -.PHONY: iac-sec -iac-sec: - @echo "[$@] :: checking the infrastructure security for $(PROJECT_ID)" - @tfsec . - @echo "[$@] :: security checked on $(PROJECT_ID)" - -# -- this target will only trigger the iac of the current parent -.PHONY: iac-version -iac-version: - @cd $(IAC_DIR) && terraform -version - -# -- this target will only trigger the iac of the current parent -.PHONY: iac-deploy -iac-deploy: iac-clean $(TF_PLAN) - @echo "[$@] :: applying the infrastructure for $(PROJECT_ID)" - @set -euo pipefail; \ - cd $(IAC_DIR) && terraform apply -auto-approve -input=false $(shell basename $(TF_PLAN)); - @echo "[$@] :: infrastructure applied on $(PROJECT_ID)" - -# -- this target re-initializes the git working tree removing untracked and ignored files -.PHONY: reinit -reinit: - @rm -rf $(IAC_DIR).terraform* $(IAC_DIR)terraform.tfstate* $(IAC_DIR)tfplan diff --git a/README.md b/README.md index 4fae162..8867b38 100644 --- a/README.md +++ b/README.md @@ -52,25 +52,7 @@ poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload ### Docker deployment -Simple docker image to build dbt-serverless for local or cloud run testing (for example). - -```docker -ARG build_for=linux/amd64 - -FROM --platform=$build_for python:3.10-slim-bullseye - -ARG DBT_PROJECT -ARG PROFILES_DIR - -WORKDIR /usr/app - -RUN pip install --no-cache-dir --upgrade pip && \ - pip install --no-cache-dir dbt-serverless - -COPY ${DBT_PROJECT}/ ${PROFILES_DIR}/profiles.yml ${DBT_PROJECT}/ - -ENTRYPOINT ["uvicorn", "dbt_serverless.main:app", "--host", "0.0.0.0", "--port", "8080"] -``` +Simple docker image to build dbt-serverless for local or cloud run testing: [example Dockerfile](examples/Dockerfile.dbt). If you're not on a Google product (like Cloud Run), you will need to specify google creds at docker runtime. @@ -80,6 +62,6 @@ For example you can add these cli parameters at runtime, if you're testing and d -v /gcp/config/logs \ --env CLOUDSDK_CONFIG=/gcp/config \ --env GOOGLE_APPLICATION_CREDENTIALS=/gcp/config/application_default_credentials.json \ - --env GOOGLE_CLOUD_PROJECT=$(PROJECT_ID) \ + --env GOOGLE_CLOUD_PROJECT=$(YOUR_PROJECT_ID) \ ``` diff --git a/dbt_serverless/main.py b/dbt_serverless/main.py index 2c74f24..44e052e 100644 --- a/dbt_serverless/main.py +++ b/dbt_serverless/main.py @@ -14,6 +14,7 @@ app = FastAPI() +PROJECT = environ.get("GOOGLE_CLOUD_PROJECT") DBT_PROJECT = environ.get("DBT_PROJECT") DBT_PROFILES_DIR = environ.get("DBT_PROFILES_DIR") @@ -79,6 +80,6 @@ async def docs() -> str: logging.info("Uploading the file to GCS for static website serving") - upload_blob("dbt-static-docs-bucket", merged_content_path, "index_merged.html") + upload_blob(f"{PROJECT}-dbt-docs", merged_content_path, "index_merged.html") return "https://storage.cloud.google.com/dbt-static-docs-bucket/index_merged.html" diff --git a/examples/.env.template b/examples/.env.template new file mode 100644 index 0000000..7424362 --- /dev/null +++ b/examples/.env.template @@ -0,0 +1,7 @@ +GOOGLE_CLOUD_PROJECT=dbt-serverless +REGION=europe-west9 +REPOSITORY_ID=gcr_artifact_repo_name + +DBT_PROJECT=dbt_project +DBT_DATASET=warehouse +DBT_PROFILES_DIR=. diff --git a/examples/Dockerfile.dbt b/examples/Dockerfile.dbt new file mode 100644 index 0000000..9dc0070 --- /dev/null +++ b/examples/Dockerfile.dbt @@ -0,0 +1,18 @@ +FROM ghcr.io/dbt-labs/dbt-bigquery:1.3.0 + +ARG DBT_PROJECT +ARG DBT_DATASET +ARG DBT_PROFILES_DIR + +ENV DBT_PROJECT=${DBT_PROJECT} +ENV DBT_DATASET=${DBT_DATASET} +ENV DBT_PROFILES_DIR=${DBT_PROFILES_DIR} + +WORKDIR / usr/app + +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir dbt-serverless + +COPY ${DBT_PROJECT}/ ${DBT_PROFILES_DIR}/profiles.yml ${DBT_PROJECT}/ + +ENTRYPOINT ["uvicorn", "dbt_serverless.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..6597ff7 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,60 @@ +Dockerfiles, Terraform and Pulumi files to test and deploy your dbt-serverless in no time. + +## Docker commands to test your own image and dbt project locally + +In your project, you could use this `Dockerfile.dbt` file and with few environment variables set (see the `.env.template` file to fill your own `.env` file), you would only need to do the following : + +```bash +export $(grep -v '^#' .env | xargs) +``` + +```bash +docker build . \ + -t $(DBT_PROJECT) \ + -f dbt.Dockerfile \ + --build-arg DBT_PROJECT=$(DBT_PROJECT) \ + --build-arg DBT_DATASET=$(DBT_DATASET) \ + --build-arg DBT_PROFILES_DIR=$(DBT_PROJECT) +``` + +```bash +docker run \ + --rm \ + --interactive \ + --tty \ + -p 8080:8080 \ + -v "$(HOME)/.config/gcloud:/gcp/config:ro" \ + -v /gcp/config/logs \ + --env CLOUDSDK_CONFIG=/gcp/config \ + --env GOOGLE_APPLICATION_CREDENTIALS=/gcp/config/application_default_credentials.json \ + --env GOOGLE_CLOUD_PROJECT=$(GOOGLE_CLOUD_PROJECT) \ + $(DBT_PROJECT) +``` + +## Deploy it on GCP + +Here we provide a `main.tf` example to deploy your dbt-serverless in an already setup GCP Project with APIs enabled and a repository in Artifact Registry. You're just gonna need 3 terraform variables (but you might already use them in your project): +- var.project +- var.region +- var.repository_id + +If it's not setup, you could just git clone the entire repository and use make commands to deploy the iac from scratch. In this case, look for the `.env.template` at the root directory and `make all`. + +Anyway! + +Build the image and tag it: + +```bash +docker build . \ + -f Dockerfile.dbt \ + --tag $REGION-docker.pkg.dev/$GOOGLE_CLOUD_PROJECT/$REPOSITORY_ID/dbt-serverless:latest \ + --build-arg DBT_PROJECT=$DBT_PROJECT \ + --build-arg DBT_DATASET=$DBT_DATASET \ + --build-arg DBT_PROFILES_DIR=$DBT_PROJECT +``` + +Push it to Google Cloud Artifact Registry repository: + +```bash +docker push $REGION-docker.pkg.dev/$GOOGLE_CLOUD_PROJECT/$REPOSITORY_ID/dbt-serverless:latest +``` diff --git a/examples/main.tf b/examples/main.tf new file mode 100644 index 0000000..cc84a9d --- /dev/null +++ b/examples/main.tf @@ -0,0 +1,56 @@ +locals { + dbt_roles = toset(["bigquery.dataEditor", "bigquery.user"]) +} +resource "google_service_account" "dbt_sa" { + account_id = "dbt-runner" + project = var.project + display_name = "dbt Service Account" + description = "dbt service account" +} + +resource "google_project_iam_member" "sa_iam_dbt" { + for_each = local.dbt_roles + project = var.project + role = "roles/${each.key}" + member = "serviceAccount:${google_service_account.dbt_sa.email}" +} + +resource "google_cloud_run_service" "dbt_serverless" { + provider = google-beta + project = var.project + location = var.region + name = "dbt-serverless" + + template { + metadata { + annotations = { + "run.googleapis.com/execution-environment" : "gen2" + } + } + spec { + containers { + image = "${var.region}-docker.pkg.dev/${var.project}/${var.repository_id}/dbt-serverless:latest" + env { + name = "GOOGLE_CLOUD_PROJECT" + value = var.project + } + resources { + limits = { + "cpu" = "1000m" + memory = "2048Mi" + } + } + } + service_account_name = google_service_account.dbt_sa.email + } + } + + traffic { + percent = 100 + latest_revision = true + } +} + +output "dbt_serverless_url" { + value = google_cloud_run_service.dbt_serverless.status[0].url +} diff --git a/includes/python.mk b/includes/python.mk new file mode 100644 index 0000000..3ffb031 --- /dev/null +++ b/includes/python.mk @@ -0,0 +1,49 @@ +check: poetry-lock clean-code clean-coverage quality test poetry-build docker-build +test: prepare-test poetry-test + +quality: prepare-quality poetry-quality +run: prepare-run poetry-run +docker: docker-build docker-run + +prepare-quality: + @poetry install --only nox,fmt,lint,type_check,docs + +poetry-quality: + @poetry run nox -s fmt_check + @poetry run nox -s lint + @poetry run nox -s type_check + @poetry run nox -s docs + +poetry-lock: + @poetry env use python3.9 + @poetry install + @poetry lock --check + +clean-coverage: + @rm -f .coverage* + @rm -f coverage.xml + +prepare-test: + @poetry install --only nox + +poetry-test: + @poetry run nox -s test-3.9 + @rm -f .coverage* + @rm -f coverage.xml + @poetry run nox -s test-3.10 + +clean-code: + @poetry run isort . + @poetry run black . + +poetry-build: + @poetry build + +prepare-run: + @poetry install --only main + +poetry-run: + @poetry run uvicorn dbt_serverless.main:app --host 0.0.0.0 --port 8080 --reload + +poetry-pulumi: + @poetry run python -m iac-pulumi.main diff --git a/includes/terraform.mk b/includes/terraform.mk new file mode 100644 index 0000000..513defe --- /dev/null +++ b/includes/terraform.mk @@ -0,0 +1,107 @@ +# ---------------------------------------------------------------------------------------- # +# -- < IaC > -- +# ---------------------------------------------------------------------------------------- # +# -- terraform variables declaration +IAC_DIR = iac/ +DBT_DIR = $(DBT_PROJECT)/ +TF_DIR = $(IAC_DIR).terraform/ +TF_INIT = $(TF_DIR)terraform.tfstate +TF_VARS = $(IAC_DIR)terraform.tfvars +TF_PLAN = $(IAC_DIR)tfplan +TF_STATE = $(wildcard $(IAC_DIR)*.tfstate $(TF_DIR)*.tfstate) +TF_FILES = $(wildcard $(IAC_DIR)*.tf) + +.PHONY: iac-clean +iac-clean: ## Clean the local terraform infrastructure + @echo "[$@] :: cleaning the infrastructure intermediary files" + @rm -fr $(TF_PLAN) $(TF_VARS); + @if [ ! -f $(IAC_DIR).iac-env ] || [ $$(cat $(IAC_DIR).iac-env || echo -n) != $(PROJECT_ID) ]; then \ + echo "[$@] :: env has changed, removing also $(TF_DIR) and $(IAC_DIR).terraform.lock.hcl"; \ + rm -rf $(TF_DIR) $(IAC_DIR).terraform.lock.hcl; \ + fi; + + @echo "[$@] :: infrastructure cleaning DONE" + +# -- this target will initialize the terraform initialization +.PHONY: iac-init +iac-init: $(TF_INIT) ## Procede to the terraform initialization +$(TF_INIT): + @set -euo pipefail; \ + if [ ! -d $(TF_DIR) ]; then \ + function remove_me() { if (( $$? != 0 )); then rm -fr $(TF_DIR); fi; }; \ + trap remove_me EXIT; \ + echo "[iac-init] :: initializing terraform"; \ + echo "$(PROJECT_ID)" > $(IAC_DIR).iac-env; \ + cd $(IAC_DIR) && terraform init \ + -backend-config=bucket=$(DEPLOY_BUCKET) \ + -backend-config=prefix=terraform-state/$(ENV) \ + -input=false; \ + else \ + echo "[iac-init] :: terraform already initialized"; \ + fi; + +# -- internal definition for easing changes +define HERE_TF_VARS +project = "$(PROJECT_ID)" +zone = "$(ZONE)" +region = "$(REGION)" +env = "$(ENV)" +repository_id = "$(REPOSITORY_ID)" +github_owner = "$(GITHUB_OWNER)" +github_repo = "$(GITHUB_REPO)" +github_token = "$(GITHUB_TOKEN)" +pypi_token = "$(PYPI_TOKEN)" +codecov_token = "$(CODECOV_TOKEN)" +endef +export HERE_TF_VARS + +# -- this target will create the terraform.tfvars file +.PHONY: iac-prepare +iac-prepare: $(TF_VARS) ## Prepares the terraform infrastructure by create the variable files +$(TF_VARS): $(TF_INIT) + @echo "[iac-prepare] :: generation of $(TF_VARS) file"; + @echo "$$HERE_TF_VARS" > $(TF_VARS); + @echo "[iac-prepare] :: generation of $(TF_VARS) file DONE."; + +# -- this target will create the tfplan file whenever the variables file and any *.tf +# file have changed +.PHONY: iac-plan +iac-plan-clean: + @rm -f $(TF_PLAN) + +iac-plan: iac-plan-clean $(TF_PLAN) ## Produce the terraform plan to visualize what will be changed in the infrastructure +$(TF_PLAN): $(TF_VARS) $(TF_FILES) + @echo "[iac-plan] :: planning the iac in $(PROJECT_ID)"; + @set -euo pipefail; \ + cd $(IAC_DIR) && terraform plan \ + -var-file $(shell basename $(TF_VARS)) \ + -out=$(shell basename $(TF_PLAN)); + @echo "[iac-plan] :: planning the iac for $(PROJECT_ID) DONE."; + +.PHONY: iac-validate +iac-validate: ## Validate the infrastructure + @echo "[$@] :: validating the infrastructure for $(PROJECT_ID)" + @set -euo pipefail; \ + cd $(IAC_DIR) && terraform validate; + @echo "[$@] :: infrastructure validated on $(PROJECT_ID)" + +.PHONY: iac-sec +iac-sec: ## Check the security of the infrastructure + @echo "[$@] :: checking the infrastructure security for $(PROJECT_ID)" + @tfsec . + @echo "[$@] :: security checked on $(PROJECT_ID)" + +.PHONY: iac-version +iac-version: ## Check the terraform version + @cd $(IAC_DIR) && terraform -version + +.PHONY: iac-deploy +iac-deploy: iac-clean $(TF_PLAN) ## Proceeds to the application of the terraform infrastructure + @echo "[$@] :: applying the infrastructure for $(PROJECT_ID)" + @set -euo pipefail; \ + cd $(IAC_DIR) && terraform apply -auto-approve -input=false $(shell basename $(TF_PLAN)); + @echo "[$@] :: infrastructure applied on $(PROJECT_ID)" + +.PHONY: reinit +reinit: ## Remove untracked files from the current git repository + @rm -rf $(IAC_DIR).terraform* $(IAC_DIR)terraform.tfstate* $(IAC_DIR)tfplan