diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..38231ac9c --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,42 @@ +name: docs +on: + push: + branches: + - master + - main + - setup-mkdocs # TODO: remove me after initial setup +permissions: + contents: write +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Configure Git Credentials + run: | + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + key: mkdocs-material-${{ env.cache_id }} + path: ~/.cache + restore-keys: | + mkdocs-material- + - run: | + pip install mkdocs-material + pip install mkdocs-git-revision-date-localized-plugin + pip install mkdocs-git-authors-plugin + pip install mike + - run: mkdocs gh-deploy --force + +# mike deploy -u 0.1 latest committoo +# do that at releases? +# also get a dev tag? with commits on master + +# mike set-default --push latest once at setup + +# mike deploy --push --update-aliases 0.1 latest diff --git a/deploy/infrastructure/modules/terraform-aws-dss/DNS.md b/deploy/infrastructure/modules/terraform-aws-dss/DNS.md deleted file mode 100644 index c0749cf1f..000000000 --- a/deploy/infrastructure/modules/terraform-aws-dss/DNS.md +++ /dev/null @@ -1,57 +0,0 @@ -# Setup DNS - -This page describes the options and steps required to setup DNS for a DSS deployment. - -## Terraform managed - -If your DNS zone is managed on the same account, it is possible to instruct terraform to create and manage -it with the rest of the infrastructure. - -- **For Elastic Kubernetes Service (AWS)**, create the zone in your aws account and set the `aws_route53_zone_id` - variable with the zone id. Entries will be automatically created by terraform. - Note that the domain or the sub-domain managed by the zone must be properly delegated by the parent domain. - See instructions for [subdomains delegation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/CreatingNewSubdomain.html#UpdateDNSParentDomain) - - -## Manual setup - -If DNS entries are managed manually, set them up manually using the following steps: - -1. Retrieve IP addresses and expected hostnames: `terraform output` - Example of expected output: - ``` - crdb_addresses = [ - { - "address" = "34.65.15.23" - "expected_dns" = "0.interuss.example.com" - }, - { - "address" = "34.65.146.56" - "expected_dns" = "1.interuss.example.com" - }, - { - "address" = "34.65.191.145" - "expected_dns" = "2.interuss.example.com" - }, - ] - gateway_address = { - "address" = "35.186.236.146" - "expected_dns" = "dss.interuss.example.com" - "certificate_validation_dns" = [ - { - "managed_by_terraform" = false - "name" = "_6e246283563dcf58e7ed.interuss.example.com." - "records" = [ - "_6e246283563dcf58e7ed.xxxxx.acm-validations.aws.", - ] - "type" = "CNAME" - }, - ] - } - ``` -2. Create the following DNS A entries to point to the static ips: - - `crdb_addresses[*].expected_dns` - - `gateway_address.expected_dns` - -3. Create the entries for SSL certificate validation according to the information provided - in `gateway_address.certificate_validation_dns`. \ No newline at end of file diff --git a/deploy/infrastructure/modules/terraform-aws-dss/README.md b/deploy/infrastructure/modules/terraform-aws-dss/README.md deleted file mode 100644 index 191b3f712..000000000 --- a/deploy/infrastructure/modules/terraform-aws-dss/README.md +++ /dev/null @@ -1,78 +0,0 @@ -# terraform-aws-dss - -This terraform module creates a Kubernetes cluster in Amazon Web Services using the Elastic Kubernetes Service (EKS) -and generates the tanka files to deploy a DSS instance. - - -## Getting started - -### Prerequisites - -Download & install the following tools to your workstation: - -1. Install [terraform](https://developer.hashicorp.com/terraform/downloads). -2. Install tools from [Prerequisites](../../../../build/README.md) -3. Install provider specific tools: - 1. [Amazon Web Services](./README.md#amazon-web-services) - - -#### Amazon Web Services - -1. Install and initialize [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions). - 1. Confirm successful installation with `aws --version`. -2. If you don't have an account, sign-up: https://aws.amazon.com/free/ -3. Configure terraform to connect to AWS using your account. - 1. We recommend to create an AWS_PROFILE using for instance `aws configure --profile aws-interuss-dss` - Before running `terraform` commands, run once in your shell: `export AWS_PROFILE=aws-interuss-dss` - Other methods are described here: https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration - - -### Deployment of the Kubernetes cluster - -1. Create a new folder in `/deploy/infrastructure/personal/` named, for instance, `terraform-aws-dss-dev`. -2. Copy main.tf, output.tf and variables.gen.tf to the new folder. -3. Copy `terraform.dev.example.tfvars` and rename to `terraform.tfvars` -4. Check that your new directory contains the following files: - - main.tf - - output.tf - - terraform.tfvars - - variables.gen.tf -5. Set the variables in `terraform.tfvars` according to your environment. See [TFVARS.gen.md](TFVARS.gen.md) for variables descriptions. -6. In the new directory (ie /deploy/infrastructure/personal/terraform-aws-dss-dev), initialize terraform: `terraform init`. -7. Run `terraform plan` to check that the configuration is valid. It will display the resources which will be provisioned. -8. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.) -9. Configure the DNS resolution according to these instructions: [Setup DNS](DNS.md) - - -## Deployment of the DSS services - -During the successful run, the terraform job has created a new [workspace](../../../../build/workspace/) -for the cluster. The new workspace name corresponds to the cluster context. The cluster context -can be retrieved by running `terraform output` in your infrastructure folder (ie /deploy/infrastructure/personal/terraform-aws-dss-dev). - -It contains scripts to operate the cluster and setup the services. - -1. Go to the new workspace `/build/workspace/${cluster_context}`. -2. Run `./get-credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. -3. If using CockroachDB: - 1. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. - 1. Deploy the certificates using `./apply-certs.sh`. -4. If using Yugabyte: - 1. Generate the certificates using `./dss-certs.sh init` - 1. If joining a cluster, check `dss-certs.sh`'s [help](../../../operations/certificates-management/README.md) to add others CA in your pool and share your CA with others pools members. - 1. Deploy the certificates using `./dss-certs.sh apply`. -5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) -6. Wait for services to initialize: - - On AWS, load balancers and certificates are created by Kubernetes Operators. Therefore, it may take few minutes (~5min) to get the services up and running and generate the certificate. To track this progress, go to the following pages and check that: - - On the [EKS page](https://eu-west-1.console.aws.amazon.com/eks/home), the status of the kubernetes cluster should be `Active`. - - On the [EC2 page](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:), the load balancers (1 for the gateway, 1 per cockroach nodes) are in the state `Active`. -7. Verify that basic services are functioning by navigating to https://your-gateway-domain.com/healthy. - - -## Clean up - -1. Note that the following operations can't be reverted and all data will be lost. -2. To delete all resources, run `tk delete .` in the workspace folder. -3. Make sure that all [load balancers](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:) and [target groups](https://eu-west-1.console.aws.amazon.com/ec2/home#TargetGroups:) have been deleted from the AWS region before next step. -4. `terraform destroy` in your infrastructure folder. -5. On the [EBS page](https://eu-west-1.console.aws.amazon.com/ec2/home#Volumes:), make sure to manually clean up the persistent storage. Note that the correct AWS region shall be selected. diff --git a/deploy/infrastructure/modules/terraform-google-dss/DNS.md b/deploy/infrastructure/modules/terraform-google-dss/DNS.md deleted file mode 100644 index 056b956b4..000000000 --- a/deploy/infrastructure/modules/terraform-google-dss/DNS.md +++ /dev/null @@ -1,40 +0,0 @@ -# Setup DNS - -This page describes the options and steps required to setup DNS for a DSS deployment. - -## Terraform managed - -If your DNS zone is managed on the same account, it is possible to instruct terraform to create and manage -it with the rest of the infrastructure. - -- **For Google Cloud Engine**, configure the zone in your google account and set the `google_dns_managed_zone_name` - variable the zone name. Zones can be listed by running `gcloud dns managed-zones list`. Entries will be - automatically created by terraform. - -## Manual setup - -If DNS entries are managed manually, set them up manually using the following steps: - -1. Retrieve IP addresses and expected hostnames: `terraform output` - Example of expected output: - ``` - crdb_addresses = [ - { - "address" = "34.65.15.23" - "expected_dns" = "0.interuss.example.com" - }, - { - "address" = "34.65.146.56" - "expected_dns" = "1.interuss.example.com" - }, - { - "address" = "34.65.191.145" - "expected_dns" = "2.interuss.example.com" - }, - ] - gateway_address = { - "address" = "35.186.236.146" - "expected_dns" = "dss.interuss.example.com" - } - -2. Create the related DNS A entries to point to the static ips. diff --git a/deploy/infrastructure/modules/terraform-google-dss/README.md b/deploy/infrastructure/modules/terraform-google-dss/README.md deleted file mode 100644 index 5070da4fb..000000000 --- a/deploy/infrastructure/modules/terraform-google-dss/README.md +++ /dev/null @@ -1,79 +0,0 @@ -# terraform-google-dss - -This terraform module creates a Kubernetes cluster in Google Cloud Engine and generates -the tanka files to deploy a DSS instance. - -## Getting started - -### Prerequisites -Download & install the following tools to your workstation: - -1. Install [terraform](https://developer.hashicorp.com/terraform/downloads). -2. Install tools from [Prerequisites](../../../../build/README.md) -3. Install provider specific tools: - 1. [Google Cloud Engine](./README.md#google-cloud-engine) - -#### Google Cloud Engine - -1. Install and initialize [Google Cloud CLI](https://cloud.google.com/sdk/docs/install-sdk). - 1. Confirm successful installation with `gcloud version`. -2. Check that the DSS project is correctly selected: gcloud config list project - 1. Set another one if needed using: `gcloud config set project $GOOGLE_PROJECT_NAME` -3. Enable the following API using [Google Cloud CLI](https://cloud.google.com/endpoints/docs/openapi/enable-api#gcloud): - 1. `compute.googleapis.com` - 2. `container.googleapis.com` - 3. If you want to manage DNS entries with terraform: `dns.googleapis.com` -4. Install the auth plugin to connect to kubernetes: `gcloud components install gke-gcloud-auth-plugin` -5. Run `gcloud auth application-default login` to generate credentials to call Google Cloud Platform APIs. - 1. If the result of performing the authorization indicates 404 in the browser, check whether a local dummy-oauth instance is running (using port 8085). Stop the dummy-oauth instance if it is running. - -### Deployment of the Kubernetes cluster - -1. Create a new folder in `/deploy/infrastructure/personal/` named for instance `terraform-google-dss-dev`. -2. Copy main.tf, output.tf and variables.gen.tf to the new folder. (Note that the modules can be added to existing projects) -3. Copy `terraform.dev.example.tfvars` and rename to `terraform.tfvars` -4. Check that your new directory contains the following files: - - main.tf - - output.tf - - terraform.tfvars - - variables.gen.tf -5. Set the variables in `terraform.tfvars` according to your environment. See [TFVARS.gen.md](TFVARS.gen.md) for variables descriptions. -6. In the new directory (ie /deploy/infrastructure/personal/terraform-google-dss-dev), initialize terraform: `terraform init`. -7. Run `terraform plan` to check that the configuration is valid. It will display the resources which will be provisioned. -8. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.) -9. Configure the DNS resolution to the public ip addresses. DNS entries can be either managed manually or -handled by terraform depending on the cloud provider. See [DNS](DNS.md) for details. - -## Deployment of the DSS services - -During the successful run, the terraform job has created a new [workspace](../../../../build/workspace/) -for the new cluster. The new workspace name corresponds to the cluster context. The cluster context -can be retrieved by running `terraform output` in your infrastructure folder (ie /deploy/infrastructure/personal/terraform-google-dss-dev). - -It contains scripts to operate the cluster and setup the services. - -1. Go to the new workspace `/build/workspace/${cluster_context}`. -2. Run `./get-credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. -3. If using CockroachDB: - 1. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. - 1. Deploy the certificates using `./apply-certs.sh`. -4. If using Yugabyte: - 1. Generate the certificates using `./dss-certs.sh init` - 1. If joining a cluster, check `dss-certs.sh`'s [help](../../../operations/certificates-management/README.md) to add others CA in your pool and share your CA with others pools members. - 1. Deploy the certificates using `./dss-certs.sh apply`. -5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) -6. Wait for services to initialize: - - On Google Cloud, the highest-latency operation is provisioning of the HTTPS certificate which generally takes 10-45 minutes. To track this progress: - - Go to the "Services & Ingress" left-side tab from the Kubernetes Engine page. - - Click on the https-ingress item (filter by just the cluster of interest if you have multiple clusters in your project). - - Under the "Ingress" section for Details, click on the link corresponding with "Load balancer". - - Under Frontend for Details, the Certificate column for HTTPS protocol will have an icon next to it which will change to a green checkmark when provisioning is complete. - - Click on the certificate link to see provisioning progress. - - If everything indicates OK and you still receive a cipher mismatch error message when attempting to visit /healthy, wait an additional 5 minutes before attempting to troubleshoot further. -7. Verify that basic services are functioning by navigating to https://your-gateway-domain.com/healthy. - -## Clean up - -To delete all resources, run `terraform destroy`. Note that this operation can't be reverted and all data will be lost. - -For Google Cloud Engine, make sure to manually clean up the persistent storage: https://console.cloud.google.com/compute/disks diff --git a/deploy/architecture.md b/docs/architecture.md similarity index 100% rename from deploy/architecture.md rename to docs/architecture.md diff --git a/docs/build.md b/docs/build.md new file mode 100644 index 000000000..7da9965de --- /dev/null +++ b/docs/build.md @@ -0,0 +1,582 @@ +# Deploying a DSS instance + +## Deployment options + +This document describes how to deploy a production-style DSS instance to +interoperate with other DSS instances in a DSS pool. + +To run a local DSS instance for testing, evaluation, or development, see +[dev/standalone_instance.md](dev/standalone_instance.md). + +To create a local DSS instance with multi-node CRDB cluster, see [dev/mutli_node_local_dss.md](dev/mutli_node_local_dss.md). + +To create or join a pool consisting of multiple interoperable DSS instances, see +[information on pooling](../deploy/operations/pooling.md). + +## Glossary + +- DSS Region - A region in which a single, unified airspace representation is + presented by one or more interoperable DSS instances, each instance typically + operated by a separate organization. A specific environment (for example, + "production" or "staging") in a particular DSS Region is called a "pool". +- DSS instance - a single logical replica in a DSS pool. + +## Preface + +This doc describes a procedure for deploying the DSS and its dependencies +(namely CockroachDB) via Kubernetes. The use of Kubernetes is not a requirement, +and a DSS instance can join a CRDB cluster constituting a DSS pool as long as it +meets the [CockroachDB requirements below](#cockroachdb-requirements). + +## Prerequisites + +Download & install the following tools to your workstation: + +- If deploying on Google Cloud, + [install Google Cloud SDK](https://cloud.google.com/sdk/install) + - Confirm successful installation with `gcloud version` + - Run `gcloud init` to set up a connection to your account. + - `kubectl` can be installed from `gcloud` instead of via the method below. +- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) to + interact with kubernetes + - Confirm successful installation with `kubectl version --client` (should + succeed from any working directory). + - Note that kubectl can alternatively be installed via the Google Cloud SDK + `gcloud` shell if using Google Cloud. +- [Install tanka](https://tanka.dev/install) + - On Linux, after downloading the binary per instructions, run + `sudo chmod +x /usr/local/bin/tk` + - Confirm successful installation with `tk --version` +- [Install Docker](https://docs.docker.com/get-docker/). + - Confirm successful installation with `docker --version` +- [Install CockroachDB](https://www.cockroachlabs.com/get-cockroachdb/) to + generate CockroachDB certificates. + - These instructions assume CockroachDB Core. + - You may need to run `sudo chmod +x /usr/local/bin/cockroach` after + completing the installation instructions. + - Confirm successful installation with `cockroach version` +- If developing the DSS codebase, + [install Golang](https://golang.org/doc/install) + - Confirm successful installation with `go version` +- Optionally install [Jsonnet](https://github.com/google/jsonnet) if editing + the jsonnet templates. + +## Docker images + +The application logic of the DSS is located in core-service which is provided in +a Docker image which is built locally and then pushed to a Docker registry of +your choice. All major cloud providers have a docker registry service, or you +can set up your own. + +To use the prebuilt InterUSS Docker images (without building them yourself), use +`docker.io/interuss/dss` for `VAR_DOCKER_IMAGE_NAME`. + +To build these images (and, optionally, push them to a docker registry): + +1. Set the environment variable `DOCKER_URL` to your docker registry url +endpoint. + + - For Google Cloud, `DOCKER_URL` should be set similarly to as described + [here](https://cloud.google.com/container-registry/docs/pushing-and-pulling#tag_the_local_image_with_the_registry_name), + like `gcr.io/your-project-id` (do not include the image name; + it will be appended by the build script) + + - For Amazon Web Services, `DOCKER_URL` should be set similarly to as described + [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html), + like `${aws_account_id}.dkr.ecr.${region}.amazonaws.com/` (do not include the image name; + it will be appended by the build script) + +1. Ensure you are logged into your docker registry service. + + - For Google Cloud, + [these](https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper) + are the recommended instructions (`gcloud auth configure-docker`). + Ensure that + [appropriate permissions are enabled](https://cloud.google.com/container-registry/docs/access-control). + + - For Amazon Web Services, create a private repository by following the instructions + [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/repository-create.html), then login + as described [here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). + +1. Use the [`build.sh` script](./build.sh) in this directory to build and push + an image tagged with the current date and git commit hash. + +1. Note the VAR_* value printed at the end of the script. + +### Access to private repository + +See below the description of `VAR_DOCKER_IMAGE_PULL_SECRET` to configure authentication. + +## Deploying a DSS instance via Kubernetes + +This section discusses deploying a Kubernetes service manually, although you can deploy +a DSS instance however you like as long as it meets the CockroachDB requirements +above. You can do this on any supported +[cloud provider](https://kubernetes.io/docs/concepts/cluster-administration/cloud-providers/) +or even on your own infrastructure. Consult the Kubernetes documentation for +your chosen provider. + +To instead deploy infrastructure using terraform, see the [terraform infrastructure deployment page](../deploy/infrastructure/README.md). + +If you can augment this documentation with specifics for another cloud provider, +a PR to that effect would be greatly appreciated. + +1. Create a new Kubernetes cluster. We recommend a new cluster for each DSS + instance. A reasonable cluster name might be `dss-us-prod-e4a` (where `e4a` + is a zone identifier abbreviation), `dss-ca-staging`, + `dss-mx-integration-sae1a`, etc. The name of this cluster will be combined + with other information by Kubernetes to generate a longer cluster context + ID. + + - On Google Cloud, the recommended procedure to create a cluster is: + - In Google Cloud Platform, go to the Kubernetes Engine page and under + Clusters click Create cluster. + - Name the cluster appropriately; e.g., `dss-us-prod` + - Select Zonal and [a compute-zone appropriate to your + geography](https://cloud.google.com/compute/docs/regions-zones#available) + - For the "default-pool" node pool: + - Enter 3 for number of nodes. + - In the "Nodes" bullet under "default-pool", select N2 series and + n2-standard-4 for machine type. + - In the "Networking" bullet under "Clusters", ensure "Enable [VPC + -native traffic](https://cloud.google.com/kubernetes-engine/docs/how-to/alias-ips)" + is checked. + +1. Make sure correct cluster context is selected by printing the context + name to the console: `kubectl config current-context` + + - Record this value and use it for `$CLUSTER_CONTEXT` below; perhaps: + `export CLUSTER_CONTEXT=$(kubectl config current-context)` + + - On Google Cloud, first configure kubectl to interact with the cluster + created above with + [these instructions](https://cloud.google.com/kubernetes-engine/docs/quickstart). + Specifically: + - `gcloud config set project your-project-id` + - `gcloud config set compute/zone your-compute-zone` + - `gcloud container clusters get-credentials your-cluster-name` + +1. Ensure the desired namespace is selected; the recommended + namespace is simply `default` with one cluster per DSS instance. Print the + the current namespaces with `kubectl get namespace`. Use the current + namespace as the value for `$NAMESPACE` below; perhaps use an environment + variable for convenience: `export NAMESPACE=`. + + It may be useful to create a `login.sh` file with content like that shown + below and `source login.sh` when working with this cluster. + + GCP: + ```shell + #!/bin/bash + + export CLUSTER_NAME= + export REGION= + gcloud config set project + gcloud config set compute/zone $REGION-a + gcloud container clusters get-credentials $CLUSTER_NAME + export CLUSTER_CONTEXT=$(kubectl config current-context) + export NAMESPACE=default + export DOCKER_URL=docker.io/interuss + echo "Current CLUSTER_CONTEXT is $CLUSTER_CONTEXT + ``` + +1. Create static IP addresses: one for the Core Service ingress, and one + for each CockroachDB node if you want to be able to interact with other + DSS instances. + + - If using Google Cloud, the Core Service ingress needs to be created as + a "Global" IP address, but the CRDB ingresses as "Regional" IP addresses. + IPv4 is recommended as IPv6 has not yet been tested. Follow + [these instructions](https://cloud.google.com/compute/docs/ip-addresses/reserve-static-external-ip-address#reserve_new_static) + to reserve the static IP addresses. Specifically (replacing + CLUSTER_NAME as appropriate since static IP addresses are defined at + the project level rather than the cluster level), e.g.: + + - `gcloud compute addresses create ${CLUSTER_NAME}-backend --global --ip-version IPV4` + - `gcloud compute addresses create ${CLUSTER_NAME}-crdb-0 --region $REGION` + - `gcloud compute addresses create ${CLUSTER_NAME}-crdb-1 --region $REGION` + - `gcloud compute addresses create ${CLUSTER_NAME}-crdb-2 --region $REGION` + +1. Link static IP addresses to DNS entries. + + - Your CockroachDB nodes should have a common hostname suffix; e.g., + `*.db.interuss.com`. Recommended naming is + `0.db.yourdeployment.yourdomain.com`, + `1.db.yourdeployment.yourdomain.com`, etc. + + - If using Google Cloud, see + [these instructions](https://cloud.google.com/dns/docs/quickstart#create_a_new_record) + to create DNS entries for the static IP addresses created above. To list + the IP addresses, use `gcloud compute addresses list`. + +1. (Only if you use CockroachDB) Use [`make-certs.py` script](./make-certs.py) to create certificates for + the CockroachDB nodes in this DSS instance: + + ./make-certs.py --cluster $CLUSTER_CONTEXT --namespace $NAMESPACE + [--node-address
...] + [--ca-cert-to-join ] + + 1. `$CLUSTER_CONTEXT` is the name of the cluster (see step 2 above). + + 1. `$NAMESPACE` is the namespace for this DSS instance (see step 3 above). + + 1. `Each ADDRESS` is the DNS entry for a CockroachDB node that will use the + certificates generated by this command. This is usually just the nodes + constituting this DSS instance, though if you maintain multiple DSS + instances in a single pool, the separate instances may share + certificates. Note that `--node-address` must include all the hostnames + and/or IP addresses that other CockroachDB nodes will use to connect to + your nodes (the nodes using these certificates). Wildcard notation is + supported, so you can use `*...com>`. If following + the recommendations above, use a single ADDRESS similar to + `*.db.yourdeployment.yourdomain.com`. The ADDRESS entries should be + separated by spaces. + + 1. If you are pooling with existing DSS instance(s) you need their CA + public cert (ca.crt), which will be concatenated with yours. Set + `--ca-cert-to-join` to a `ca.crt` file. Reach out to existing operators + to request their public cert. If not joining an existing pool, omit + this argument. + + 1. Note: If you are creating multiple DSS instances at once, and joining + them together you likely want to copy the nth instance's `ca.crt` into + the rest of the instances, such that ca.crt is the same across all + instances. + +1. (Only if you use Yugabyte) Use [`dss-certs.py` script](../deploy/operations/certificates-management/README.md) to create certificates for the Yugabyte nodes in this DSS instance. + +1. If joining an existing DSS pool, share ca.crt with the DSS instance(s) you + are trying to join, and have them apply the new ca.crt, which now contains + both your instance's and the original instance's public certs, to enable + secure bi-directional communication. Each original DSS instance, upon + receipt of the combined ca.crt from the joining instance, should perform the + actions below. While they are performing those actions, you may continue + with the instructions. + + 1. If you use CockroachDB: + + 1. Overwrite its existing ca.crt with the new ca.crt provided by the DSS + instance joining the pool. + 1. Upload the new ca.crt to its cluster using + `./apply-certs.sh $CLUSTER_CONTEXT $NAMESPACE` + 1. Restart their CockroachDB pods to recognize the updated ca.crt: + `kubectl rollout restart statefulset/cockroachdb --namespace $NAMESPACE` + 1. Inform you when their CockroachDB pods have finished restarting + (typically around 10 minutes) + + 1. If you use Yugabyte + + 1. Share your CA with `./dss-certs.py get-ca` + 1. Add others CAs of the pool with `./dss-certs.py add-pool-ca` + 1. Upload the new CAs to its cluster using + `./dss-certs.py apply` + 1. Restart their Yugabyte pods to recognize the updated ca.crt: + `kubectl rollout restart statefulset/yb-master --namespace $NAMESPACE` + `kubectl rollout restart statefulset/yb-tserver --namespace $NAMESPACE` + 1. Inform you when their Yugabyte pods have finished restarting + (typically around 10 minutes) + +1. Ensure the Docker images are built according to the instructions in the + [previous section](#docker-images). + +1. From this working directory, + `cp -r ../deploy/services/tanka/examples/minimum/* workspace/$CLUSTER_CONTEXT`. Note that + the `workspace/$CLUSTER_CONTEXT` folder should have already been created + by the `make-certs.py` script. + Replace the imports at the top of `main.jsonnet` to correctly locate the files: + ``` + local dss = import '../../../deploy/services/tanka/dss.libsonnet'; + local metadataBase = import '../../../deploy/services/tanka/metadata_base.libsonnet'; + ``` + +1. If providing a .pem file directly as the public key to validate incoming + access tokens, copy it to [dss/build/jwt-public-certs](./jwt-public-certs). + Public key specification by JWKS is preferred; if using the JWKS approach + to specify the public key, skip this step. + +1. Edit `workspace/$CLUSTER_CONTEXT/main.jsonnet` and replace all `VAR_*` + instances with appropriate values: + + 1. `VAR_NAMESPACE`: Same `$NAMESPACE` used in the make-certs.py (and + apply-certs.sh) scripts. + + 1. `VAR_CLUSTER_CONTEXT`: Same $CLUSTER_CONTEXT used in the `make-certs.py` + and `apply-certs.sh` scripts. + + 1. `VAR_ENABLE_SCD`: Set this boolean true to enable strategic conflict + detection functionality (currently an R&D project tracking an initial + draft of the upcoming ASTM standard). + + 1. `VAR_LOCALITY`: Unique name for your DSS instance. Currently, we + recommend "_", and the `=` character is not + allowed. However, any unique (among all other participating DSS + instances) value is acceptable. + + 1. `VAR_DB_HOSTNAME_SUFFIX`: The domain name suffix shared by all of your + CockroachDB nodes. For instance, if your CRDB nodes were addressable at + `0.db.example.com`, `1.db.example.com`, and `2.db.example.com`, then + VAR_DB_HOSTNAME_SUFFIX would be `db.example.com`. + + 1. `VAR_DATASTORE`: Datastore to use. Can be set to 'cockroachdb' or 'yugabyte'. + + 1. `VAR_CRDB_DOCKER_IMAGE_NAME`: Docker image of cockroach db pods. Until + DSS v0.16, the recommended CockroachDB image name is `cockroachdb/cockroach:v21.2.7`. + From DSS v0.17, the recommended CockroachDB version is `cockroachdb/cockroach:v24.1.3`. + + 1. `VAR_CRDB_NODE_IPn`: IP address (**numeric**) of nth CRDB node (add more + entries if you have more than 3 CRDB nodes). Example: `1.1.1.1` + + 1. `VAR_SHOULD_INIT`: Set to `false` if joining an existing pool, `true` + if creating the first DSS instance for a pool. When set `true`, this + can initialize the data directories on your cluster, and prevent you + from joining an existing pool. + + 1. `VAR_EXTERNAL_CRDB_NODEn`: Fully-qualified domain name of existing CRDB + nodes if you are joining an existing pool. If more than three are + available, add additional entries. If not joining an existing pool, + comment out this `JoinExisting:` line. + + - You should supply a minimum of 3 seed nodes to every CockroachDB node. + These 3 nodes should be the same for every node (ie: every node points + to node 0, 1, and 2). For external DSS instances you should point to a + minimum of 3, or you can use a loadbalanced hostname or IP address of + other DSS instances. You should do this for every DSS instance in the + pool, including newly joined instances. See CockroachDB's note on the + [join flag](https://www.cockroachlabs.com/docs/stable/start-a-node.html#flags). + + 1. `VAR_YUGABYTE_DOCKER_IMAGE_NAME`: Docker image of Yugabyte db pods. + Shall be set to at least `yugabytedb/yugabyte:2.25.1.0-b381` + + 1. `VAR_YUGABYTE_MASTER_IPn`: IP address (**numeric**) of nth Yugabyte + master node (add more entries if you have more than 3 nodes). + Example: `1.1.1.1` + + 1. `VAR_YUGABYTE_TSERVER_IPn`: IP address (**numeric**) of nth Yugabyte + tserver node (add more entries if you have more than 3 nodes). + Example: `1.1.1.1` + + 1. `VAR_YUGABYTE_MASTER_ADDRESSn`: List of addresses of Yugabyte master + nodes in the DSS pool. Must be accessible from all master/tserver nodes + and identical in a cluster. Example: `["0.master.db.uss1.example.com", "1.master.db.uss1.example.com", "3.master.db.uss1.example.com", "0.master.db.uss2.example.com", "1.master.db.uss2.example.com", "3.master.db.uss2.example.com"]` + You may remove this setting if you only have a simple 3-nodes local cluster. + + 1. `VAR_YUGABYTE_MASTER_RPC_BIND_ADDRESSES`: Bind address for yugabyte + master node. May use `${HOSTNAME}`, `${NAMESPACE}` or `${HOSTNAMENO}` + to use respectively hostname, namespace or number of the node. + Example: `${HOSTNAMENO}.master.db.uss1.example.com` + You may remove this setting if you only have a simple 3-nodes local cluster. + + 1. `VAR_YUGABYTE_MASTER_BROADCAST_ADDRESSES`: Broadcast address for yugabyte + master node. May use `${HOSTNAME}`, `${NAMESPACE}` or `${HOSTNAMENO}` + to use respectively hostname, namespace or number of the node. + Example: `${HOSTNAMENO}.master.db.uss1.example.com:7100` + You may remove this setting if you only have a simple 3-nodes local cluster. + + 1. `VAR_YUGABYTE_TSERVER_RPC_BIND_ADDRESSES`: Bind address for yugabyte + tserver node. May use `${HOSTNAME}`, `${NAMESPACE}` or `${HOSTNAMENO}` + to use respectively hostname, namespace or number of the node. + Example: `${HOSTNAMENO}.tserver.db.uss1.example.com` + You may remove this setting if you only have a simple 3-nodes local cluster. + + 1. `VAR_YUGABYTE_TSERVER_BROADCAST_ADDRESSES`: Broadcast address for yugabyte + tserver node. May use `${HOSTNAME}`, `${NAMESPACE}` or `${HOSTNAMENO}` + to use respectively hostname, namespace or number of the node. + Example: `${HOSTNAMENO}.tserver.db.uss1.example.com:9100` + You may remove this setting if you only have a simple 3-nodes local cluster. + + 1. `VAR_YUGABYTE_FIX_27367_ISSUE`: Fix issue [27367](https://github.com/yugabyte/yugabyte-db/issues/27367) + To make the fix working, RPC bind and broadcast addresses must be set to + the same, public value on where the master / tserver node is accessible. + + 1. `VAR_YUGABYTE_LIGHT_RESOURCES`: Use light resources in term of CPU/Memory + for Yugabyte nodes. You may use that for development purposes, to deploy + a Yugabyte in a small cluster to save costs and resources. + + 1. `VAR_YUGABYTE_PLACEMENT_CLOUD`: Yugabyte placement's cloud value, for + master and tserver nodes. + Example: `cloud-1` + + 1. `VAR_YUGABYTE_PLACEMENT_REGION`: Yugabyte placement's region value, for + master and tserver nodes. + Example: `uss-1` + + 1. `VAR_YUGABYTE_PLACEMENT_ZONE`: Yugabyte placement's zone value, for + master and tserver nodes. + Example: `zone-1` + + 1. `VAR_STORAGE_CLASS`: Kubernetes Storage Class to use for CockroachDB, + Yugabyte and Prometheus volumes. You can check your cluster's possible + values with `kubectl get storageclass`. If you're not sure, each cloud + provider has some default storage classes that should work: + - Google Cloud: `standard` + - Azure: `default` + - AWS: `gp2` + + 1. `VAR_INGRESS_NAME`: If using Google Kubernetes Engine, set this to the + the name of the core-service static IP address created above (e.g., + `CLUSTER_NAME-backend`). + + 1. `VAR_DOCKER_IMAGE_NAME`: Full name of the docker image built in the + section above. `build.sh` prints this name as the last thing it does + when run with `DOCKER_URL` set. It should look something like + `gcr.io/your-project-id/dss:2020-07-01-46cae72cf` if you built the image + yourself, or `docker.io/interuss/dss` if using the InterUSS image + without `build.sh`. + + - Note that `VAR_DOCKER_IMAGE_NAME` is used in two places. + + 1. `VAR_DOCKER_IMAGE_PULL_SECRET`: Secret name of the credentials to access + the image registry. If the image specified in VAR_DOCKER_IMAGE_NAME does not require + authentication to be pulled, then do not populate this instance and do not uncomment + the line containing it. You can use the following command to store the credentials + as kubernetes secret: + + > kubectl create secret -n VAR_NAMESPACE docker-registry VAR_DOCKER_IMAGE_PULL_SECRET \ + --docker-server=DOCKER_REGISTRY_SERVER \ + --docker-username=DOCKER_USER \ + --docker-password=DOCKER_PASSWORD \ + --docker-email=DOCKER_EMAIL + + For docker hub private repository, use `docker.io` as `DOCKER_REGISTRY_SERVER` and an + [access token](https://hub.docker.com/settings/security) as `DOCKER_PASSWORD`. + + 1. `VAR_APP_HOSTNAME`: Fully-qualified domain name of your Core Service + ingress endpoint. For example, `dss.example.com`. + + 1. `VAR_PUBLIC_ENDPOINT`: URL to publicly access your Core Service + ingress endpoint. For example, `https://dss.example.com`. Only for versions >=0.21. + + 1. `VAR_PUBLIC_KEY_PEM_PATH`: If providing a .pem file directly as the + public key to validate incoming access tokens, specify the name of this + .pem file here as `/jwt-public-certs/YOUR-KEY-NAME.pem` replacing + YOUR-KEY-NAME as appropriate. For instance, if using the provided + [`us-demo.pem`](./jwt-public-certs/us-demo.pem), use the path + `/jwt-public-certs/us-demo.pem`. Note that your .pem file must have + been copied into [`jwt-public-certs`](./jwt-public-certs) in an earlier + step, or mounted at runtime using a volume. + + - If providing an access token public key via JWKS, provide a blank + string for this parameter. + + 1. `VAR_JWKS_ENDPOINT`: If providing the access token public key via JWKS, + specify the JWKS endpoint here. Example: + `https://auth.example.com/.well-known/jwks.json` + + - If providing a .pem file directly as the public key to valid incoming access tokens, provide a blank string for this parameter. + + 1. `VAR_JWKS_KEY_ID`: If providing the access token public key via JWKS, + specify the `kid` (key ID) of they appropriate key in the JWKS file + referenced above. + + - If providing a .pem file directly as the public key to valid incoming access tokens, provide a blank string for this parameter. + + - If you are only turning up a single DSS instance for development, you + may optionally change `single_cluster` to `true`. + + 1. `VAR_SSL_POLICY`: When deploying on Google Cloud, a [ssl policy](https://cloud.google.com/load-balancing/docs/ssl-policies-concepts) + can be applied to the DSS Ingress. This can be used to secure the TLS connection. + Follow the [instructions](https://cloud.google.com/load-balancing/docs/use-ssl-policies) to create the Global SSL Policy and + replace VAR_SSL_POLICY variable with its name. `RESTRICTED` profile is recommended. + Leave it empty if not applicable. + + 1. `VAR_ENABLE_SCHEMA_MANAGER`: Set this to true to enable the schema manager jobs. + It is required to perform schema upgrades. Note that it is automatically enabled when `VAR_SHOULD_INIT` is true. + + +1. Edit workspace/$CLUSTER_CONTEXT/spec.json and replace all VAR_* + instances with appropriate values: + + 1. VAR_API_SERVER: Determine this value with the command: + + `echo $(kubectl config view -o jsonpath="{.clusters[?(@.name==\"$CLUSTER_CONTEXT\")].cluster.server}")` + + - Note that `$CLUSTER_CONTEXT` should be replaced with your actual + `CLUSTER_CONTEXT` value prior to executing the above command if you + have not defined a `CLUSTER_CONTEXT` environment variable. + + 1. VAR_NAMESPACE: See previous section. + +1. Use the [`apply-certs.sh` script](apply-certs.sh) to create secrets on the + Kubernetes cluster containing the certificates and keys generated in the + previous step. + + ./apply-certs.sh $CLUSTER_CONTEXT $NAMESPACE + +1. Run `tk apply workspace/$CLUSTER_CONTEXT` to apply it to the + cluster. + + - If you are joining an existing pool, do not execute this command until the + the existing DSS instances all confirm that their CockroachDB pods have + finished their rolling restarts. + +1. Wait for services to initialize. Verify that basic services are functioning + by navigating to https://your-domain.example.com/healthy. + + - On Google Cloud, the highest-latency operation is provisioning of the + HTTPS certificate which generally takes 10-45 minutes. To track this + progress: + - Go to the "Services & Ingress" left-side tab from the Kubernetes Engine + page. + - Click on the `https-ingress` item (filter by just the cluster of + interest if you have multiple clusters in your project). + - Under the "Ingress" section for Details, click on the link corresponding + with "Load balancer". + - Under Frontend for Details, the Certificate column for HTTPS protocol + will have an icon next to it which will change to a green checkmark when + provisioning is complete. + - Click on the certificate link to see provisioning progress. + - If everything indicates OK and you still receive a cipher mismatch error + message when attempting to visit /healthy, wait an additional 5 minutes + before attempting to troubleshoot further. + +1. If joining an existing pool, share your CRDB node addresses with the + operators of the existing DSS instances. They will add these node addresses + to JoinExisting where `VAR_CRDB_EXTERNAL_NODEn` is indicated in the minimum + example, and then update their deployment: + + `tk apply workspace/$CLUSTER_CONTEXT` + +## Pooling + +See [the pooling documentation](../deploy/operations/pooling.md). + +## Tools + +See [operations monitoring documentation](../deploy/operations/monitoring.md). + +## Troubleshooting + +See [Troubleshooting in `deploy/operations`](../deploy/operations/troubleshooting.md). + +## Upgrading Database Schemas + +All schemas-related files are in `db_schemas` directory. Any changes you +wish to make to the database schema should be done in their respective database +folders. The files are applied in sequential numeric steps from the current +version M to the desired version N. + +For the first-ever run during the CRDB cluster initialization, the db-manager +will run once to bootstrap and bring the database up to date. To upgrade +existing clusters you will need to: + +### If performing this operation on the original cluster +1. Update the `desired_xyz_db_version` field in `main.jsonnet` +2. Delete the existing db-manager job in your k8s cluster +3. Redeploy the newly configured db-manager with `tk apply -t job/`. It should automatically up/down grade your database schema to your desired version. + +### If performing this operation on any other cluster + +1. Create `workspace/$CLUSTER_CONTEXT_schema_manager` in this (build) directory. + +1. From this (build) working directory, + `cp -r ../deploy/services/tanka/examples/schema_manager/* workspace/$CLUSTER_CONTEXT_schema_manager`. + +1. Edit `workspace/$CLUSTER_CONTEXT_schema_manager/main.jsonnet` and replace all `VAR_*` + instances with appropriate values where applicable as explained in the above section. + +1. Run `tk apply workspace/$CLUSTER_CONTEXT_schema_manager` + +### Garbage collector job +Only since commit [c789b2b](https://github.com/interuss/dss/commit/c789b2b4a9fa5fb651d202da0a3abc02a03c15d2) on Aug 25, 2020 will the DSS enable automatic garbage collection of records by tracking which DSS instance is responsible for garbage collection of the record. Expired records added with a DSS deployment running code earlier than this must be manually removed. + +The Garbage collector job runs every 30 minute to delete records in RID tables that records' endtime is 30 minutes less than current time. If the event takes a long time and takes longer than 30 minutes (previous job is still running), the job will skip a run until the previous job completes. diff --git a/deploy/README.md b/docs/index.md similarity index 100% rename from deploy/README.md rename to docs/index.md diff --git a/docs/infrastructure/cloud/cleanup.md b/docs/infrastructure/cloud/cleanup.md new file mode 100644 index 000000000..3da2e1095 --- /dev/null +++ b/docs/infrastructure/cloud/cleanup.md @@ -0,0 +1,13 @@ +# Clean up +Note that the following operations can't be reverted and all data will be lost. + +## AWS +3. To delete all resources, run `tk delete .` in the workspace folder. +3. Make sure that all [load balancers](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:) and [target groups](https://eu-west-1.console.aws.amazon.com/ec2/home#TargetGroups:) have been deleted from the AWS region before next step. +4. `terraform destroy` in your infrastructure folder. +5. On the [EBS page](https://eu-west-1.console.aws.amazon.com/ec2/home#Volumes:), make sure to manually clean up the persistent storage. Note that the correct AWS region shall be selected. + +## GCP +To delete all resources, run `terraform destroy`. Note that this operation can't be reverted and all data will be lost. + +For Google Cloud Engine, make sure to manually clean up the persistent storage: https://console.cloud.google.com/compute/disks diff --git a/docs/infrastructure/cloud/cluster.md b/docs/infrastructure/cloud/cluster.md new file mode 100644 index 000000000..d257f716d --- /dev/null +++ b/docs/infrastructure/cloud/cluster.md @@ -0,0 +1,15 @@ +# Deploy Kubernetes cluster + +1. Create a new folder in `/deploy/infrastructure/personal/` named for instance `terraform-cloud-dss-dev`. +2. Copy main.tf, output.tf and variables.gen.tf to the new folder. +3. Copy `terraform.dev.example.tfvars` and rename to `terraform.tfvars` +4. Check that your new directory contains the following files: + - main.tf + - output.tf + - terraform.tfvars + - variables.gen.tf +5. Set the variables in `terraform.tfvars` according to your environment. See [TFVARS.gen.md](TFVARS.gen.md) for variables descriptions. + TODO: differnt depending on provider +6. In the new directory (i.e. /deploy/infrastructure/personal/terraform-cloud-dss-dev), initialize terraform: `terraform init`. +7. Run `terraform plan` to check that the configuration is valid. It will display the resources which will be provisioned. +8. Run `terraform apply` to deploy the cluster. (This operation may take up to 15 min.) diff --git a/docs/infrastructure/cloud/dns.md b/docs/infrastructure/cloud/dns.md new file mode 100644 index 000000000..921eda058 --- /dev/null +++ b/docs/infrastructure/cloud/dns.md @@ -0,0 +1,90 @@ +# Setup DNS + +This page describes the options and steps required to setup DNS for a DSS deployment. + +## Terraform managed + +If your DNS zone is managed on the same account, it is possible to instruct terraform to create and manage +it with the rest of the infrastructure. + +=== "AWS" + **For Elastic Kubernetes Service (AWS)**, create the zone in your aws account and set the `aws_route53_zone_id` + variable with the zone id. Entries will be automatically created by terraform. + Note that the domain or the sub-domain managed by the zone must be properly delegated by the parent domain. + See instructions for [subdomains delegation](https://docs.aws.amazon.com/Route53/latest/DeveloperGuide/CreatingNewSubdomain.html#UpdateDNSParentDomain) + +=== "GCP" + **For Google Cloud Engine**, configure the zone in your google account and set the `google_dns_managed_zone_name` + variable the zone name. Zones can be listed by running `gcloud dns managed-zones list`. Entries will be + automatically created by terraform. + + +## Manual setup + +If DNS entries are managed manually, set them up manually using the following steps: + +### AWS +1. Retrieve IP addresses and expected hostnames: `terraform output` + Example of expected output: + ``` + crdb_addresses = [ + { + "address" = "34.65.15.23" + "expected_dns" = "0.interuss.example.com" + }, + { + "address" = "34.65.146.56" + "expected_dns" = "1.interuss.example.com" + }, + { + "address" = "34.65.191.145" + "expected_dns" = "2.interuss.example.com" + }, + ] + gateway_address = { + "address" = "35.186.236.146" + "expected_dns" = "dss.interuss.example.com" + "certificate_validation_dns" = [ + { + "managed_by_terraform" = false + "name" = "_6e246283563dcf58e7ed.interuss.example.com." + "records" = [ + "_6e246283563dcf58e7ed.xxxxx.acm-validations.aws.", + ] + "type" = "CNAME" + }, + ] + } + ``` +2. Create the following DNS A entries to point to the static ips: + - `crdb_addresses[*].expected_dns` + - `gateway_address.expected_dns` + +3. Create the entries for SSL certificate validation according to the information provided + in `gateway_address.certificate_validation_dns`. + + +### GCP +1. Retrieve IP addresses and expected hostnames: `terraform output` + Example of expected output: + ``` + crdb_addresses = [ + { + "address" = "34.65.15.23" + "expected_dns" = "0.interuss.example.com" + }, + { + "address" = "34.65.146.56" + "expected_dns" = "1.interuss.example.com" + }, + { + "address" = "34.65.191.145" + "expected_dns" = "2.interuss.example.com" + }, + ] + gateway_address = { + "address" = "35.186.236.146" + "expected_dns" = "dss.interuss.example.com" + } + ``` +2. Create the related DNS A entries to point to the static ips. diff --git a/docs/infrastructure/cloud/dss.md b/docs/infrastructure/cloud/dss.md new file mode 100644 index 000000000..ce54ad7b6 --- /dev/null +++ b/docs/infrastructure/cloud/dss.md @@ -0,0 +1,31 @@ +# Deploy DSS services + +During the successful run, the terraform job has created a new [workspace](../../../../build/workspace/) +for the cluster. The new workspace name corresponds to the cluster context. The cluster context +can be retrieved by running `terraform output` in your infrastructure folder (ie /deploy/infrastructure/personal/terraform-aws-dss-dev). + +It contains scripts to operate the cluster and setup the services. + +1. Go to the new workspace `/build/workspace/${cluster_context}`. +2. Run `./get-credentials.sh` to login to kubernetes. You can now access the cluster with `kubectl`. +3. If using CockroachDB: + 1. Generate the certificates using `./make-certs.sh`. Follow script instructions if you are not initializing the cluster. + 1. Deploy the certificates using `./apply-certs.sh`. +4. If using Yugabyte: + 1. Generate the certificates using `./dss-certs.sh init` + 1. If joining a cluster, check `dss-certs.sh`'s [help](../../../operations/certificates-management/README.md) to add others CA in your pool and share your CA with others pools members. + 1. Deploy the certificates using `./dss-certs.sh apply`. +5. Run `tk apply .` to deploy the services to kubernetes. (This may take up to 30 min) +6. Wait for services to initialize: + - On AWS, load balancers and certificates are created by Kubernetes Operators. Therefore, it may take few minutes (~5min) to get the services up and running and generate the certificate. To track this progress, go to the following pages and check that: + - On the [EKS page](https://eu-west-1.console.aws.amazon.com/eks/home), the status of the kubernetes cluster should be `Active`. + - On the [EC2 page](https://eu-west-1.console.aws.amazon.com/ec2/home#LoadBalancers:), the load balancers (1 for the gateway, 1 per cockroach nodes) are in the state `Active`. + - On Google Cloud, the highest-latency operation is provisioning of the HTTPS certificate which generally takes 10-45 minutes. To track this progress: + - Go to the "Services & Ingress" left-side tab from the Kubernetes Engine page. + - Click on the https-ingress item (filter by just the cluster of interest if you have multiple clusters in your project). + - Under the "Ingress" section for Details, click on the link corresponding with "Load balancer". + - Under Frontend for Details, the Certificate column for HTTPS protocol will have an icon next to it which will change to a green checkmark when provisioning is complete. + - Click on the certificate link to see provisioning progress. + - If everything indicates OK and you still receive a cipher mismatch error message when attempting to visit /healthy, wait an additional 5 minutes before attempting to troubleshoot further. +7. Verify that basic services are functioning by navigating to https://your-gateway-domain.com/healthy. + diff --git a/docs/infrastructure/cloud/prereqs.md b/docs/infrastructure/cloud/prereqs.md new file mode 100644 index 000000000..585dd2d6a --- /dev/null +++ b/docs/infrastructure/cloud/prereqs.md @@ -0,0 +1,52 @@ +# Prerequisites + +## Terraform +Install [Terraform](https://developer.hashicorp.com/terraform/downloads). + + +## Kubernetes cluster configuration +=== "Tanka" + - [Install tanka](https://tanka.dev/install) + - On Linux, after downloading the binary per instructions, run `sudo chmod +x /usr/local/bin/tk` + - Confirm successful installation with `tk --version` + - Optionally install [Jsonnet](https://github.com/google/jsonnet) if editing the jsonnet templates. + +=== "Helm" + TBD + + +## Database configuration +=== "CockroachDB" + TBD + +=== "Yugabyte" + [Install CockroachDB](https://www.cockroachlabs.com/get-cockroachdb/) to generate CockroachDB certificates. + + - These instructions assume CockroachDB Core. + - You may need to run `sudo chmod +x /usr/local/bin/cockroach` after completing the installation instructions. + - Confirm successful installation with `cockroach version` + + +## Cloud CLI client +=== "AWS" + 1. Install and initialize [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html#getting-started-install-instructions). + 1. Confirm successful installation with `aws --version`. + 2. If you don't have an account, sign-up: https://aws.amazon.com/free/ + 3. Configure terraform to connect to AWS using your account. + 1. We recommend to create an AWS_PROFILE using for instance `aws configure --profile aws-interuss-dss` + Before running `terraform` commands, run once in your shell: `export AWS_PROFILE=aws-interuss-dss` + Other methods are described here: https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration + +=== "GCP" + 1. Install and initialize [Google Cloud CLI](https://cloud.google.com/sdk/docs/install-sdk). + 1. Confirm successful installation with `gcloud version`. + 2. Check that the DSS project is correctly selected: gcloud config list project + 1. Set another one if needed using: `gcloud config set project $GOOGLE_PROJECT_NAME` + 3. Enable the following API using [Google Cloud CLI](https://cloud.google.com/endpoints/docs/openapi/enable-api#gcloud): + 1. `compute.googleapis.com` + 2. `container.googleapis.com` + 3. If you want to manage DNS entries with terraform: `dns.googleapis.com` + 4. Install the auth plugin to connect to kubernetes: `gcloud components install gke-gcloud-auth-plugin` + 5. Run `gcloud auth application-default login` to generate credentials to call Google Cloud Platform APIs. + 1. If the result of performing the authorization indicates 404 in the browser, check whether a local dummy-oauth instance is running (using port 8085). Stop the dummy-oauth instance if it is running. + diff --git a/docs/infrastructure/dependencies/terraform-aws-kubernetes/README.md b/docs/infrastructure/dependencies/terraform-aws-kubernetes/README.md new file mode 100644 index 000000000..7c9370807 --- /dev/null +++ b/docs/infrastructure/dependencies/terraform-aws-kubernetes/README.md @@ -0,0 +1,131 @@ +# terraform-aws-kubernetes + +To deploy a complete DSS to AWS Elastic Kubernetes Service, see [terraform-aws-dss](../../modules/terraform-aws-dss). + +This folder only contains the terraform module which deploys the kubernetes cluster required to run the DSS on +Kubernetes in AWS. + +## Configuration + +See [variables.gen.tf](variables.gen.tf). + +## Design + +This module creates an EKS cluster with related worker nodes. EKS requires 2 subnets in different availability zones ( +AZ). A dedicated VPC is created to that effect. At the moment, worker nodes are deployed in the two first AZ of the +region. The following table summarizes current responsibilities for resource creation in the AWS account: + +| Resource type | Manager | +|---------------------------------------------|-----------------------------------------------------| +| VPC and Subnets | Terraform | +| Elastic IPs | Terraform | +| Network Load balancer | Kubernetes controller: aws-load-balancer-controller | +| Target groups | Kubernetes controller: aws-load-balancer-controller | +| Storage Volumes (Elastic Block Storage) | EKS add-on provisioned by terraform | +| SSL Certificates (AWS Certificates Manager) | Terraform | +| DNS | Terraform (or manual) | + + +### AWS Load Balancers and Kubernetes Services + +Load balancers are provisioned by the Kubernetes controller aws-load-balancer-controller v2.12 +with [Option A for IAM configuration](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.12/deploy/installation/#option-a-recommended-iam-roles-for-service-accounts-irsa). + +Network Load Balancers map elastic IPs to Kubernetes Services (Load Balancer). Application Load Balancers (Ingress) do +not support this feature at the moment, making impossible to anticipate DNS records inside the cluster. + +The Network Load Balancers are provisioned by the aws-load-balancer-controller. It handles the TLS termination for +the dss https service. + +Note that the load balancer is distributing the traffic to possibly multiple subnets. Be aware that it is not possible +to unassign a subnet. Target pods shall be deployed in every subnet, meaning that the pods should be properly +distributed in worker nodes and a worker node should be at least present in each subnets. + +Provisioning is done by annotating a Kubernetes Service resource. The following example deploys a simple http server: + +```yaml +--- +apiVersion: v1 +kind: Namespace +metadata: + name: example +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: example-app + namespace: example + labels: + app: example-app +spec: + replicas: 2 + selector: + matchLabels: + app: example-app + template: + metadata: + labels: + app: example-app + spec: + containers: + - name: nginx + image: public.ecr.aws/nginx/nginx:1.21 + ports: + - name: http + containerPort: 80 + imagePullPolicy: IfNotPresent +--- +apiVersion: v1 +kind: Service +metadata: + name: example-service + namespace: example + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: external + service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: ip + service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing" + service.beta.kubernetes.io/aws-load-balancer-ssl-ports: '443' + service.beta.kubernetes.io/aws-load-balancer-ssl-cert: [ CERTIFICATE_ARN ] + service.beta.kubernetes.io/aws-load-balancer-eip-allocations: [ EIP_ALLOCATION_ID1,EIP_ALLOCATION_ID2,... ] + service.beta.kubernetes.io/aws-load-balancer-name: [ LOAD_BALANCER_NAME ] + service.beta.kubernetes.io/aws-load-balancer-subnets: [ SUBNET_ID1,SUBNET_ID2,... ] +spec: + selector: + app: example-app + ports: + - port: 443 + targetPort: 80 + protocol: TCP + name: http + type: LoadBalancer + loadBalancerClass: service.k8s.aws/nlb +``` + +- [CERTIFICATE_ARN]: arn of the certificate managed by AWS Certificate Manager +- [EIP_ALLOCATION_IDx]: Elastic IP allocation id (The number of elastic IP should equal to the number of SUBNET_IDx) +- [LOAD_BALANCER_NAME]: Name of the balancer created by the controller +- [SUBNET_IDx]: Name or ID of a subnet (The number of subnets should equal to the number of EIP_ALLOCATION_IDx) + + +## Test + +`terraform apply` generates a resource specification `test-app.yml`. This file can be applied to test a http server +reachability within the deployed cluster. To apply the resources, follow the next steps: + +1. Login to the EKS cluster: `aws eks --region $AWS_REGION update-kubeconfig --name $CLUSTER_NAME` +2. Create the resources: `kubectl apply -f test-app.yml` +3. Wait (up to 5min) for the load balancer to be ready. Note that the load balancer may take few minutes to start. + Monitor the progress here until the state becomes `Active`: https://console.aws.amazon.com/ec2/home#LoadBalancers: +4. Connect to the app_hostname and contemplate the nginx default welcome page. + +### Clean up test + +Delete the resources: `kubectl delete -f test-app.yml`. + + +## Clean up infrastructure + +1. Delete all created resources from the cluster (eg. clean up test as described in the previous section.) +2. Make sure all load balancers and target groups have been removed. +3. Run `terraform destroy`. + diff --git a/docs/infrastructure/dependencies/terraform-commons-dss/README.md b/docs/infrastructure/dependencies/terraform-commons-dss/README.md new file mode 100644 index 000000000..c10eabdd9 --- /dev/null +++ b/docs/infrastructure/dependencies/terraform-commons-dss/README.md @@ -0,0 +1,11 @@ +# terraform-commons-dss + +This folder contains a terraform module which gathers resources required by all cloud providers. + +It currently consists of the automatic generation of the tanka configuration to deploy +the Kubernetes resources as well as the scripts required to generate the certificates +and operate the cluster. + +## Configuration + +See [variables.gen.tf](variables.gen.tf). diff --git a/docs/infrastructure/dependencies/terraform-google-kubernetes/README.md b/docs/infrastructure/dependencies/terraform-google-kubernetes/README.md new file mode 100644 index 000000000..fd4e86fe1 --- /dev/null +++ b/docs/infrastructure/dependencies/terraform-google-kubernetes/README.md @@ -0,0 +1,9 @@ +# terraform-google-kubernetes + +To deploy a complete DSS to Google Cloud Engine, see [terraform-google-dss](../../modules/terraform-google-dss). + +This folder only contains the terraform module which deploys the kubernetes cluster required to run the DSS on Kubernetes in Google Cloud Engine. + +## Configuration + +See [variables.gen.tf](variables.gen.tf). diff --git a/docs/infrastructure/index.md b/docs/infrastructure/index.md new file mode 100644 index 000000000..526b150de --- /dev/null +++ b/docs/infrastructure/index.md @@ -0,0 +1,34 @@ +# Infrastructure + +As a phase in [DSS deployment](..), this folder contains the terraform modules required to prepare the infrastructure to host a DSS deployment. To deploy infrastructure manually (rather than terraform, as described here), see ["Deploying a DSS instance via Kubernetes"](../../build/README.md#deploying-a-dss-instance-via-kubernetes). + +See [Services](../README.md#services) to deploy the DSS once the infrastructure is ready. + +## Modules +The [modules](modules) directory contains the terraform public modules required to prepare the infrastructure on various cloud providers. + +- [terraform-aws-dss](./modules/terraform-aws-dss/README.md): Amazon Web Services deployment +- [terraform-google-dss](./modules/terraform-google-dss/README.md): Google Cloud Engine deployment + +This terraform module creates a Kubernetes cluster in Amazon Web Services using the Elastic Kubernetes Service (EKS) +and generates the tanka files to deploy a DSS instance. +This terraform module creates a Kubernetes cluster in Google Cloud Engine and generates +the tanka files to deploy a DSS instance. + + +## Dependencies +The [dependencies](dependencies) directory contains submodules used by the public modules described above. They are not expected to be +used directly by users. Those submodules are the combination of the cloud specific dependencies `terraform-*-kubernetes` +and `terraform-common-dss`. `terraform-common-dss` module aggregates and outputs the infrastructure configuration +which can be used as input to the `Services` deployment as shown in the diagram below. + +![Infrastructure Modules](../../assets/generated/deploy_infrastructure_modules.png) + +## Local + +The [local](local) directory contains various documentation that can be used to spawn a cluster locally. + +- [minikuke](./local/minikube/README.md): Minikube local deployment + +## Utils +This [utils folder](utils) contains scripts to help manage the terraform modules and dependencies. See the README in that folder for details. diff --git a/deploy/infrastructure/local/minikube/README.md b/docs/infrastructure/local/minikube.md similarity index 100% rename from deploy/infrastructure/local/minikube/README.md rename to docs/infrastructure/local/minikube.md diff --git a/docs/infrastructure/local/prereqs.md b/docs/infrastructure/local/prereqs.md new file mode 100644 index 000000000..a7fb8ba07 --- /dev/null +++ b/docs/infrastructure/local/prereqs.md @@ -0,0 +1,41 @@ +# Prerequisites + +Download & install the following tools to your Linux workstation: + +- [Install kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) to + interact with kubernetes + - Confirm successful installation with `kubectl version --client` (should + succeed from any working directory). + +- [Install Docker](https://docs.docker.com/get-docker/). + - Confirm successful installation with `docker --version` + +- If developing the DSS codebase, + [install Golang](https://golang.org/doc/install) + - Confirm successful installation with `go version` + +- Install [minikube](https://minikube.sigs.k8s.io/docs/start/) (First step only). + + + +## Configuration +=== "Helm" + TBD + +=== "Tanka" + - [Install tanka](https://tanka.dev/install) + - On Linux, after downloading the binary per instructions, run + `sudo chmod +x /usr/local/bin/tk` + - Confirm successful installation with `tk --version` + - Optionally install [Jsonnet](https://github.com/google/jsonnet) if editing the jsonnet templates. + +## Database +=== "CockroachDB" + TBD + +=== "Yugabyte" + [Install CockroachDB](https://www.cockroachlabs.com/get-cockroachdb/) to generate CockroachDB certificates. + + - These instructions assume CockroachDB Core. + - You may need to run `sudo chmod +x /usr/local/bin/cockroach` after completing the installation instructions. + - Confirm successful installation with `cockroach version` \ No newline at end of file diff --git a/deploy/MIGRATION.md b/docs/migration.md similarity index 100% rename from deploy/MIGRATION.md rename to docs/migration.md diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..57303bb36 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,94 @@ +# yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json + +site_name: DSS Deployment Documentation +site_url: https://orbitalize.github.io/dss # TODO: adapt me +repo_url: https://github.com/Orbitalize/dss # TODO: adapt me +extra: + version: + provider: mike # TODO https://squidfunk.github.io/mkdocs-material/setup/setting-up-versioning/ + social: + - icon: fontawesome/brands/github + link: https://github.com/interuss +docs_dir: docs +#edit_uri: edit/main/docs/ # TODO: for content.action.edit, if subfolder, main is the branch +theme: + name: material + icon: + repo: fontawesome/brands/github + features: + # enhance navigation (see https://squidfunk.github.io/mkdocs-material/setup/setting-up-navigation/) + - navigation.instant + - navigation.instant.prefetch + - navigation.instant.progress + - navigation.tracking + - navigation.tabs + - navigation.sections + - navigation.path + - navigation.indexes + - toc.follow + - navigation.top + + # https://squidfunk.github.io/mkdocs-material/setup/setting-up-site-search/ + - search.suggest + + # https://squidfunk.github.io/mkdocs-material/setup/setting-up-the-header/ + # https://squidfunk.github.io/mkdocs-material/setup/setting-up-the-footer/ + - header.autohide + - navigation.footer + + # https://squidfunk.github.io/mkdocs-material/setup/adding-a-git-repository/ + - content.action.edit + - content.action.view + +plugins: + - search +# - git-revision-date-localized: # TODO: reenable me +# enable_creation_date: true +# fallback_to_build_date: true +# - git-authors # TODO: reenable me + +# https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown/ +markdown_extensions: + - admonition # enable use of call-outs: + - pymdownx.details # make call-outs collapsible + - def_list + - footnotes + - tables + - pymdownx.betterem + - pymdownx.highlight: + anchor_linenums: true + pygments_lang_class: true + linenums: true + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - pymdownx.inlinehilite + - pymdownx.smartsymbols + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + - toc: + permalink: true # displays a permalink next to headings + +nav: + - Home: + - 'index.md' + - 'architecture.md' + - 'migration.md' + - Infrastructure: + - 'infrastructure/index.md' + - Cloud deployment with Terraform: + - 'infrastructure/cloud/prereqs.md' + - 'infrastructure/cloud/cluster.md' + - 'infrastructure/cloud/dns.md' + - 'infrastructure/cloud/dss.md' + - 'infrastructure/cloud/cleanup.md' + - Local deployment with Minikube: + - 'infrastructure/local/prereqs.md' + - 'infrastructure/local/minikube.md' +# - Services: +# - Operations + +# set up this with releases: https://squidfunk.github.io/mkdocs-material/setup/setting-up-versioning/