From 4e1a253310764b6f04e8b1e7967df9b3467269f4 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Thu, 30 Apr 2026 11:09:53 +0600 Subject: [PATCH 1/6] docs(milvus): add guides and examples --- .../milvus/quickstart/distributed.yaml | 23 +++ docs/guides/milvus/README.md | 71 +++++++ docs/guides/milvus/_index.md | 10 + docs/guides/milvus/concepts/_index.md | 10 + docs/guides/milvus/concepts/catalog.md | 47 +++++ docs/guides/milvus/concepts/milvus.md | 57 ++++++ docs/guides/milvus/concepts/opsrequest.md | 42 +++++ docs/guides/milvus/configuration/_index.md | 10 + .../milvus/configuration/using-config-file.md | 136 +++++++++++++ docs/guides/milvus/custom-rbac/_index.md | 10 + .../milvus/custom-rbac/using-custom-rbac.md | 157 +++++++++++++++ docs/guides/milvus/monitoring/_index.md | 10 + docs/guides/milvus/monitoring/overview.md | 39 ++++ .../monitoring/using-builtin-prometheus.md | 146 ++++++++++++++ .../monitoring/using-prometheus-operator.md | 132 +++++++++++++ docs/guides/milvus/ops-request/_index.md | 10 + docs/guides/milvus/ops-request/overview.md | 29 +++ docs/guides/milvus/private-registry/_index.md | 10 + .../using-private-registry.md | 126 +++++++++++++ docs/guides/milvus/quickstart/_index.md | 10 + docs/guides/milvus/quickstart/quickstart.md | 93 +++++++++ docs/guides/milvus/quickstart/rbac.md | 178 ++++++++++++++++++ docs/guides/milvus/tls/_index.md | 10 + docs/guides/milvus/tls/overview.md | 39 ++++ 24 files changed, 1405 insertions(+) create mode 100644 docs/examples/milvus/quickstart/distributed.yaml create mode 100644 docs/guides/milvus/README.md create mode 100644 docs/guides/milvus/_index.md create mode 100644 docs/guides/milvus/concepts/_index.md create mode 100644 docs/guides/milvus/concepts/catalog.md create mode 100644 docs/guides/milvus/concepts/milvus.md create mode 100644 docs/guides/milvus/concepts/opsrequest.md create mode 100644 docs/guides/milvus/configuration/_index.md create mode 100644 docs/guides/milvus/configuration/using-config-file.md create mode 100644 docs/guides/milvus/custom-rbac/_index.md create mode 100644 docs/guides/milvus/custom-rbac/using-custom-rbac.md create mode 100644 docs/guides/milvus/monitoring/_index.md create mode 100644 docs/guides/milvus/monitoring/overview.md create mode 100644 docs/guides/milvus/monitoring/using-builtin-prometheus.md create mode 100644 docs/guides/milvus/monitoring/using-prometheus-operator.md create mode 100644 docs/guides/milvus/ops-request/_index.md create mode 100644 docs/guides/milvus/ops-request/overview.md create mode 100644 docs/guides/milvus/private-registry/_index.md create mode 100644 docs/guides/milvus/private-registry/using-private-registry.md create mode 100644 docs/guides/milvus/quickstart/_index.md create mode 100644 docs/guides/milvus/quickstart/quickstart.md create mode 100644 docs/guides/milvus/quickstart/rbac.md create mode 100644 docs/guides/milvus/tls/_index.md create mode 100644 docs/guides/milvus/tls/overview.md diff --git a/docs/examples/milvus/quickstart/distributed.yaml b/docs/examples/milvus/quickstart/distributed.yaml new file mode 100644 index 000000000..f0f9aa17a --- /dev/null +++ b/docs/examples/milvus/quickstart/distributed.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi diff --git a/docs/guides/milvus/README.md b/docs/guides/milvus/README.md new file mode 100644 index 000000000..ef8652c30 --- /dev/null +++ b/docs/guides/milvus/README.md @@ -0,0 +1,71 @@ +--- +title: Milvus +menu: + docs_{{ .version }}: + identifier: milvus-readme + name: Milvus + parent: milvus-guides + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +url: /docs/{{ .version }}/guides/milvus/ +aliases: + - /docs/{{ .version }}/guides/milvus/README/ +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Overview + +KubeDB supports vector database deployment with Milvus using the `Milvus` CRD. + +## Supported Milvus Features + +| Features | Availability | +|----------------------------------|:------------:| +| Standalone provisioning | ✓ | +| Distributed provisioning | ✓ | +| Monitoring | ✓ | +| TLS | No | +| Ops Requests | No | + +## Example Milvus Manifest + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +``` + +## User Guide + +- [Quickstart Milvus](/docs/guides/milvus/quickstart/quickstart.md) with KubeDB operator. +- [Milvus CRD Concept](/docs/guides/milvus/concepts/milvus.md). +- [MilvusVersion CRD Concept](/docs/guides/milvus/concepts/catalog.md). +- [MilvusOpsRequest CRD Concept](/docs/guides/milvus/concepts/opsrequest.md). +- [RBAC Quickstart](/docs/guides/milvus/quickstart/rbac.md) +- [Private Registry](/docs/guides/milvus/private-registry/using-private-registry.md) +- [Custom RBAC](/docs/guides/milvus/custom-rbac/using-custom-rbac.md) +- [Custom Configuration](/docs/guides/milvus/configuration/using-config-file.md) +- [Monitoring](/docs/guides/milvus/monitoring/overview.md) for metrics collection guidance. +- [Builtin Prometheus Monitoring](/docs/guides/milvus/monitoring/using-builtin-prometheus.md) +- [Prometheus Operator Monitoring](/docs/guides/milvus/monitoring/using-prometheus-operator.md) \ No newline at end of file diff --git a/docs/guides/milvus/_index.md b/docs/guides/milvus/_index.md new file mode 100644 index 000000000..2b29e986f --- /dev/null +++ b/docs/guides/milvus/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus +menu: + docs_{{ .version }}: + identifier: milvus-guides + name: Milvus + parent: guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/_index.md b/docs/guides/milvus/concepts/_index.md new file mode 100644 index 000000000..f522969c1 --- /dev/null +++ b/docs/guides/milvus/concepts/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Concepts +menu: + docs_{{ .version }}: + identifier: milvus-concepts + name: Concepts + parent: milvus-guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/catalog.md b/docs/guides/milvus/concepts/catalog.md new file mode 100644 index 000000000..0df3f06b5 --- /dev/null +++ b/docs/guides/milvus/concepts/catalog.md @@ -0,0 +1,47 @@ +--- +title: MilvusVersion CRD +menu: + docs_{{ .version }}: + identifier: milvus-catalog-concepts + name: MilvusVersion + parent: milvus-concepts-milvus + weight: 15 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MilvusVersion + +## What is MilvusVersion + +`MilvusVersion` is the catalog CRD that defines the Milvus engine image and related metadata for KubeDB-managed Milvus deployments. + +KubeDB uses this CRD when resolving `Milvus.spec.version`. + +## MilvusVersion Specification + +```yaml +apiVersion: catalog.kubedb.com/v1alpha1 +kind: MilvusVersion +metadata: + name: "2.6.11" +spec: + version: "2.6.11" + db: + image: "kubedb/milvus:2.6.11" + deprecated: false +``` + +## Key fields + +- `metadata.name` is referenced from `Milvus.spec.version`. +- `spec.version` is the Milvus release version. +- `spec.db.image` is the image used for Milvus components. +- `spec.deprecated` indicates whether the entry is deprecated. + +## Next Steps + +- Read the [Milvus CRD concept](/docs/guides/milvus/concepts/milvus.md). +- Run the [Milvus quickstart](/docs/guides/milvus/quickstart/quickstart.md). \ No newline at end of file diff --git a/docs/guides/milvus/concepts/milvus.md b/docs/guides/milvus/concepts/milvus.md new file mode 100644 index 000000000..b79d3f564 --- /dev/null +++ b/docs/guides/milvus/concepts/milvus.md @@ -0,0 +1,57 @@ +--- +title: Milvus CRD +menu: + docs_{{ .version }}: + identifier: milvus-concepts-milvus + name: Milvus + parent: milvus-concepts + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Milvus + +## What is Milvus + +`Milvus` is a KubeDB `CustomResourceDefinition` used to deploy and manage Milvus vector databases. + +## Milvus Spec + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +``` + +### Key fields + +- `spec.version` points to a `MilvusVersion`. +- `spec.objectStorage` is required for object data. +- `spec.topology.mode` supports `Standalone` or `Distributed`. +- `spec.topology.distributed` configures distributed roles. +- `spec.metaStorage` can configure external or managed etcd. +- `spec.storageType` and `spec.storage` define persistent data storage. +- `spec.authSecret`, `spec.configuration`, `spec.monitor`, and `spec.serviceTemplates` are optional controls. \ No newline at end of file diff --git a/docs/guides/milvus/concepts/opsrequest.md b/docs/guides/milvus/concepts/opsrequest.md new file mode 100644 index 000000000..a83fa19f8 --- /dev/null +++ b/docs/guides/milvus/concepts/opsrequest.md @@ -0,0 +1,42 @@ +--- +title: MilvusOpsRequest CRD +menu: + docs_{{ .version }}: + identifier: milvus-opsrequest-concepts + name: MilvusOpsRequest + parent: milvus-concepts-milvus + weight: 25 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MilvusOpsRequest + +## What is MilvusOpsRequest + +`MilvusOpsRequest` is the operations CRD KubeDB uses for day-2 lifecycle changes of Milvus databases when supported in a release. + +## Current support status + +Based on [new_db.md](/new_db.md), no Milvus operation types are currently listed. + +## Expected CRD shape + +```yaml +apiVersion: ops.kubedb.com/v1alpha1 +kind: MilvusOpsRequest +metadata: + name: milvus-ops-sample + namespace: demo +spec: + type: Restart + databaseRef: + name: milvus-cluster +``` + +## Next Steps + +- Track [Milvus ops overview](/docs/guides/milvus/ops-request/overview.md) for support updates. +- Use [Milvus TLS guide](/docs/guides/milvus/tls/overview.md) and [Milvus monitoring guide](/docs/guides/milvus/monitoring/overview.md) for day-2 operations. \ No newline at end of file diff --git a/docs/guides/milvus/configuration/_index.md b/docs/guides/milvus/configuration/_index.md new file mode 100644 index 000000000..0224bf33d --- /dev/null +++ b/docs/guides/milvus/configuration/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus with Custom Configuration +menu: + docs_{{ .version }}: + identifier: milvus-configuration + name: Custom Configuration + parent: milvus-guides + weight: 130 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/configuration/using-config-file.md b/docs/guides/milvus/configuration/using-config-file.md new file mode 100644 index 000000000..b14de52e0 --- /dev/null +++ b/docs/guides/milvus/configuration/using-config-file.md @@ -0,0 +1,136 @@ +--- +title: Run Milvus with Custom Configuration +menu: + docs_{{ .version }}: + identifier: milvus-using-config-file + name: Config File + parent: milvus-configuration + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Custom Configuration File + +KubeDB supports providing custom configuration for Milvus. This tutorial will show you how to use KubeDB to run Milvus with custom configuration. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +Milvus supports configuration via the `milvus.yaml` file. KubeDB takes advantage of `spec.configuration.secretName` to allow users to provide a custom `milvus.yaml` without mounting any volume into the Pod. The operator reads this Secret internally and applies the configuration automatically. + +In this tutorial, we will configure `queryNode.gracefulTime` and `dataNode.segment.maxSize` parameters. + +## Custom Configuration + +At first, let's create a custom `milvus.yaml` file: + +```yaml +queryNode: + gracefulTime: 5000 + +dataNode: + segment: + maxSize: 512 +``` + +Now, create a Secret with this configuration file. + +```bash +$ kubectl create secret generic -n demo milvus-configuration \ + --from-file=milvus.yaml=./milvus.yaml +secret/milvus-configuration created +``` + +Verify the Secret has the configuration file. + +```bash +$ kubectl get secret -n demo milvus-configuration -o yaml +apiVersion: v1 +data: + milvus.yaml: +kind: Secret +metadata: + name: milvus-configuration + namespace: demo +``` + +Now, create Milvus CRD specifying `spec.configuration.secretName` field. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: custom-milvus + namespace: demo +spec: + version: "2.4.0" + configuration: + secretName: milvus-configuration + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/configuration/milvus-configuration.yaml +milvus.kubedb.com/custom-milvus created +``` + +Now, wait for the Milvus to be ready. + +```bash +$ kubectl get milvus -n demo custom-milvus +NAME VERSION STATUS AGE +custom-milvus 2.4.0 Ready 3m +``` + +Check that the pod is running: + +```bash +$ kubectl get pod -n demo custom-milvus-0 +NAME READY STATUS RESTARTS AGE +custom-milvus-0 1/1 Running 0 3m +``` + +Now, we will verify the configuration has been applied. We will `exec` into the pod and check the configuration file. + +```bash +$ kubectl exec -it -n demo custom-milvus-0 -- cat /milvus/configs/milvus.yaml | grep -A 5 queryNode +queryNode: + gracefulTime: 5000 +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/custom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/custom-milvus + +kubectl delete -n demo secret milvus-configuration +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/custom-rbac/_index.md b/docs/guides/milvus/custom-rbac/_index.md new file mode 100644 index 000000000..d322e1d6d --- /dev/null +++ b/docs/guides/milvus/custom-rbac/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus with Custom RBAC resources +menu: + docs_{{ .version }}: + identifier: milvus-custom-rbac + name: Custom RBAC + parent: milvus-guides + weight: 90 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/custom-rbac/using-custom-rbac.md b/docs/guides/milvus/custom-rbac/using-custom-rbac.md new file mode 100644 index 000000000..33247b520 --- /dev/null +++ b/docs/guides/milvus/custom-rbac/using-custom-rbac.md @@ -0,0 +1,157 @@ +--- +title: Run Milvus with Custom RBAC resources +menu: + docs_{{ .version }}: + identifier: milvus-custom-rbac-quickstart + name: Custom RBAC + parent: milvus-custom-rbac + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Custom RBAC Resources + +KubeDB supports finer user control over role based access permissions provided to a Milvus instance. This tutorial will show you how to use KubeDB to run Milvus instance with custom RBAC resources. + +## Before You Begin + +At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +## Overview + +KubeDB allows users to provide custom RBAC resources, namely, `ServiceAccount`, `Role`, and `RoleBinding` for Milvus. This is provided via the `spec.podTemplate.spec.serviceAccountName` field in Milvus CRD. + +## Custom RBAC for Milvus + +At first, let's create a `Service Account` in `demo` namespace. + +```bash +$ kubectl create serviceaccount -n demo my-custom-serviceaccount +serviceaccount/my-custom-serviceaccount created +``` + +Now, we need to create a role that has necessary access permissions for the Milvus database named `quick-milvus`. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: my-custom-role + namespace: demo +rules: +- apiGroups: + - apps + resourceNames: + - quick-milvus + resources: + - petsets + verbs: + - get +- apiGroups: + - kubedb.com + resourceNames: + - quick-milvus + resources: + - milvuses + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-role.yaml +role.rbac.authorization.k8s.io/my-custom-role created +``` + +Now create a `RoleBinding` to bind this `Role` with the already created service account. + +```bash +$ kubectl create rolebinding my-custom-rolebinding \ + --role=my-custom-role \ + --serviceaccount=demo:my-custom-serviceaccount \ + --namespace=demo +rolebinding.rbac.authorization.k8s.io/my-custom-rolebinding created +``` + +Now, create a Milvus CRD specifying `spec.podTemplate.spec.serviceAccountName` field to `my-custom-serviceaccount`. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: quick-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + podTemplate: + spec: + serviceAccountName: my-custom-serviceaccount + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-db.yaml +milvus.kubedb.com/quick-milvus created +``` + +Check that the pod is running: + +```bash +$ kubectl get pod -n demo quick-milvus-0 +NAME READY STATUS RESTARTS AGE +quick-milvus-0 1/1 Running 0 3m +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/quick-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/quick-milvus + +kubectl delete -n demo serviceaccount my-custom-serviceaccount +kubectl delete -n demo role my-custom-role +kubectl delete -n demo rolebinding my-custom-rolebinding +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/monitoring/_index.md b/docs/guides/milvus/monitoring/_index.md new file mode 100644 index 000000000..a1c83edfb --- /dev/null +++ b/docs/guides/milvus/monitoring/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Monitoring +menu: + docs_{{ .version }}: + identifier: milvus-monitoring + name: Monitoring + parent: milvus-guides + weight: 20 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/monitoring/overview.md b/docs/guides/milvus/monitoring/overview.md new file mode 100644 index 000000000..248796e32 --- /dev/null +++ b/docs/guides/milvus/monitoring/overview.md @@ -0,0 +1,39 @@ +--- +title: Milvus Monitoring Overview +menu: + docs_{{ .version }}: + identifier: milvus-monitoring-overview + name: Overview + parent: milvus-monitoring + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus Monitoring + +This guide shows how to enable and verify monitoring for Milvus deployments. + +## Before You Begin + +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). +- Install Prometheus Operator or another compatible metrics collection stack. + +## Enable Monitoring + +Milvus supports monitoring integration through `spec.monitor`. + +- Configure monitoring agent and service monitor selectors. +- Validate scrape targets for proxy and distributed components. + +## Verify + +```bash +kubectl get milvus -n demo milvus-cluster -o yaml +kubectl get servicemonitor -A +``` + +## Next Steps + +- Review which Milvus components need dedicated scrape targets in distributed mode. +- Add dashboards for latency, indexing throughput, and node health. diff --git a/docs/guides/milvus/monitoring/using-builtin-prometheus.md b/docs/guides/milvus/monitoring/using-builtin-prometheus.md new file mode 100644 index 000000000..dc5b8dfe3 --- /dev/null +++ b/docs/guides/milvus/monitoring/using-builtin-prometheus.md @@ -0,0 +1,146 @@ +--- +title: Monitor Milvus using Builtin Prometheus Discovery +menu: + docs_{{ .version }}: + identifier: milvus-using-builtin-prometheus-monitoring + name: Builtin Prometheus + parent: milvus-monitoring + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring Milvus with Builtin Prometheus + +This tutorial will show you how to monitor Milvus database using builtin [Prometheus](https://github.com/prometheus/prometheus) scraper. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/milvus/monitoring/overview.md). + +- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy the database in `demo` namespace. + + ```bash + $ kubectl create ns monitoring + namespace/monitoring created + + $ kubectl create ns demo + namespace/demo created + ``` + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Deploy Milvus with Monitoring Enabled + +At first, let's deploy a Milvus database with monitoring enabled. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: builtin-prom-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/builtin +``` + +Here, `spec.monitor.agent: prometheus.io/builtin` specifies that we are going to monitor this server using builtin Prometheus scraper. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/monitoring/builtin-prom-milvus.yaml +milvus.kubedb.com/builtin-prom-milvus created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get milvus -n demo builtin-prom-milvus +NAME VERSION STATUS AGE +builtin-prom-milvus 2.4.0 Ready 2m +``` + +KubeDB will create a separate stats service with name `{Milvus crd name}-stats` for monitoring purpose. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=builtin-prom-milvus" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +builtin-prom-milvus ClusterIP 10.96.100.20 19530/TCP 2m +builtin-prom-milvus-stats ClusterIP 10.96.100.21 9091/TCP 90s +``` + +Let's describe the stats service: + +```bash +$ kubectl describe svc -n demo builtin-prom-milvus-stats +Name: builtin-prom-milvus-stats +Namespace: demo +Labels: app.kubernetes.io/name=milvuses.kubedb.com + app.kubernetes.io/instance=builtin-prom-milvus +Annotations: monitoring.appscode.com/agent: prometheus.io/builtin + prometheus.io/path: /metrics + prometheus.io/port: 9091 + prometheus.io/scrape: true +``` + +The service contains the following annotations which are used by builtin Prometheus to discover the endpoint: + +``` +prometheus.io/path: /metrics +prometheus.io/port: 9091 +prometheus.io/scrape: true +``` + +Configure your Prometheus to scrape metrics from the `monitoring` namespace service discovery: + +```yaml +scrape_configs: +- job_name: kubedb-milvuses + honor_labels: true + kubernetes_sd_configs: + - role: endpoints + namespaces: + names: + - demo + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/builtin-prom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/builtin-prom-milvus + +kubectl delete ns demo +kubectl delete ns monitoring +``` diff --git a/docs/guides/milvus/monitoring/using-prometheus-operator.md b/docs/guides/milvus/monitoring/using-prometheus-operator.md new file mode 100644 index 000000000..a0726fd45 --- /dev/null +++ b/docs/guides/milvus/monitoring/using-prometheus-operator.md @@ -0,0 +1,132 @@ +--- +title: Monitor Milvus using Prometheus Operator +menu: + docs_{{ .version }}: + identifier: milvus-using-prometheus-operator-monitoring + name: Prometheus Operator + parent: milvus-monitoring + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring Milvus using Prometheus Operator + +[Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) provides simple and Kubernetes native way to deploy and configure Prometheus, Alertmanager and related monitoring components. This tutorial will show you how to use the Prometheus operator to monitor Milvus database deployed with KubeDB. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/milvus/monitoring/overview.md). + +- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy the database in `demo` namespace. + + ```bash + $ kubectl create ns monitoring + namespace/monitoring created + + $ kubectl create ns demo + namespace/demo created + ``` + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Deploy Milvus with Monitoring Enabled + +At first, let's deploy a Milvus database with monitoring enabled. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: coreos-prom-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + interval: 10s +``` + +Here, +- `spec.monitor.agent: prometheus.io/operator` indicates that we are going to monitor this server using Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels` specifies that KubeDB should create `ServiceMonitor` with these labels. +- `spec.monitor.prometheus.serviceMonitor.interval` indicates that the Prometheus should scrape metrics from this database with 10 seconds interval. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/monitoring/coreos-prom-milvus.yaml +milvus.kubedb.com/coreos-prom-milvus created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get milvus -n demo coreos-prom-milvus +NAME VERSION STATUS AGE +coreos-prom-milvus 2.4.0 Ready 2m +``` + +KubeDB will create a `ServiceMonitor` CRD in the same namespace as the Milvus database. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +coreos-prom-milvus 2m +``` + +Let's verify the `ServiceMonitor` has the right label to be discovered by Prometheus: + +```bash +$ kubectl get servicemonitor -n demo coreos-prom-milvus -o yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + release: prometheus + name: coreos-prom-milvus + namespace: demo +spec: + endpoints: + - honorLabels: true + interval: 10s + path: /metrics + port: metrics + namespaceSelector: + matchNames: + - demo + selector: + matchLabels: + app.kubernetes.io/instance: coreos-prom-milvus +``` + +Now, if we go to the Prometheus dashboard, we should see the target being scraped. + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/coreos-prom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/coreos-prom-milvus + +kubectl delete ns demo +kubectl delete ns monitoring +``` diff --git a/docs/guides/milvus/ops-request/_index.md b/docs/guides/milvus/ops-request/_index.md new file mode 100644 index 000000000..9d0b5d697 --- /dev/null +++ b/docs/guides/milvus/ops-request/_index.md @@ -0,0 +1,10 @@ +--- +title: Ops Request +menu: + docs_{{ .version }}: + identifier: milvus-ops-request + name: Ops Request + parent: milvus-guides + weight: 30 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/ops-request/overview.md b/docs/guides/milvus/ops-request/overview.md new file mode 100644 index 000000000..ae0b7e728 --- /dev/null +++ b/docs/guides/milvus/ops-request/overview.md @@ -0,0 +1,29 @@ +--- +title: Milvus Ops Request Overview +menu: + docs_{{ .version }}: + identifier: milvus-ops-request-overview + name: Overview + parent: milvus-ops-request + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus Ops Request + +This page tracks Milvus operations supported by the new database matrix used for these guides. + +## Before You Begin + +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). +- Confirm the current support list in [new_db.md](/new_db.md) for your release. + +## Current Status + +Based on [new_db.md](/new_db.md), no Milvus ops request types are listed yet. + +## What to Check Next + +- Watch future KubeDB releases for Milvus `OpsRequest` support. +- Use the TLS and monitoring guides for common day-2 platform tasks. diff --git a/docs/guides/milvus/private-registry/_index.md b/docs/guides/milvus/private-registry/_index.md new file mode 100644 index 000000000..c202e6450 --- /dev/null +++ b/docs/guides/milvus/private-registry/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus using Private Registry +menu: + docs_{{ .version }}: + identifier: milvus-private-registry + name: Private Registry + parent: milvus-guides + weight: 120 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/private-registry/using-private-registry.md b/docs/guides/milvus/private-registry/using-private-registry.md new file mode 100644 index 000000000..3ebbb5ab4 --- /dev/null +++ b/docs/guides/milvus/private-registry/using-private-registry.md @@ -0,0 +1,126 @@ +--- +title: Run Milvus using Private Registry +menu: + docs_{{ .version }}: + identifier: milvus-using-private-registry + name: Quickstart + parent: milvus-private-registry + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Private Docker Registry + +KubeDB supports using private Docker registries. This tutorial will show you how to run KubeDB managed Milvus database using private Docker images. + +## Before You Begin + +At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +## Prepare Private Docker Registry + +- You will also need a docker private [registry](https://docs.docker.com/registry/) or [private repository](https://docs.docker.com/docker-hub/repos/#private-repositories). + +- Push the required images from KubeDB's [Docker hub account](https://hub.docker.com/r/kubedb/) into your private registry. For Milvus, push the `DB_IMAGE` of the following MilvusVersions, where `deprecated` is not true. + + ```bash + $ kubectl get milvusversions -o=custom-columns=NAME:.metadata.name,VERSION:.spec.version,DB_IMAGE:.spec.db.image,DEPRECATED:.spec.deprecated + NAME VERSION DB_IMAGE DEPRECATED + 2.4.0 2.4.0 kubedb/milvus:2.4.0 + ``` + +## Create ImagePullSecret + +Run the following command to create an image pull secret for your private Docker registry: + +```bash +$ kubectl create secret docker-registry -n demo myregistrykey \ + --docker-server=DOCKER_REGISTRY_SERVER \ + --docker-username=DOCKER_USER \ + --docker-email=DOCKER_EMAIL \ + --docker-password=DOCKER_PASSWORD +secret/myregistrykey created +``` + +## Install KubeDB Operator + +Follow the steps to [install KubeDB operator](/docs/setup/README.md) properly in cluster so that it points to the DOCKER_REGISTRY you wish to pull images from. + +## Create MilvusVersion CRD + +Create a MilvusVersion CRD specifying images from your private registry. Replace `PRIVATE_REGISTRY` with your private registry. + +```yaml +apiVersion: catalog.kubedb.com/v1alpha1 +kind: MilvusVersion +metadata: + name: "2.4.0" +spec: + db: + image: PRIVATE_REGISTRY/milvus:2.4.0 + version: "2.4.0" +``` + +```bash +$ kubectl apply -f pvt-milvusversion.yaml +milvusversion.catalog.kubedb.com/2.4.0 created +``` + +## Deploy Milvus from Private Registry + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: pvt-reg-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + podTemplate: + spec: + imagePullSecrets: + - name: myregistrykey + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/private-registry/pvt-reg-milvus.yaml +milvus.kubedb.com/pvt-reg-milvus created +``` + +Check that the Milvus is in Running state: + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=pvt-reg-milvus" +NAME READY STATUS RESTARTS AGE +pvt-reg-milvus-0 1/1 Running 0 3m +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/pvt-reg-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/pvt-reg-milvus + +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/quickstart/_index.md b/docs/guides/milvus/quickstart/_index.md new file mode 100644 index 000000000..753aefff9 --- /dev/null +++ b/docs/guides/milvus/quickstart/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Quickstart +menu: + docs_{{ .version }}: + identifier: milvus-quickstart + name: Quickstart + parent: milvus-guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/quickstart.md b/docs/guides/milvus/quickstart/quickstart.md new file mode 100644 index 000000000..4a7c4480c --- /dev/null +++ b/docs/guides/milvus/quickstart/quickstart.md @@ -0,0 +1,93 @@ +--- +title: Milvus Quickstart +menu: + docs_{{ .version }}: + identifier: milvus-quickstart-overview + name: Overview + parent: milvus-quickstart + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Running Milvus + +This tutorial shows how to run a Milvus database with KubeDB. + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus/quickstart](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus/quickstart). + +## Before You Begin + +- Prepare a Kubernetes cluster and `kubectl`. +- Install KubeDB from [/docs/setup/README.md](/docs/setup/README.md). +- This tutorial uses `docs/examples/milvus/quickstart/distributed.yaml` as the working example manifest. +- Create namespace: + +```bash +kubectl create ns demo +``` + +## Check Available MilvusVersion + +```bash +kubectl get milvusversions +``` + +## Check Object Storage Secret + +Milvus requires an object storage backend for metadata and data files. Make sure the secret referenced by the example manifest exists before creating the database. + +```bash +kubectl get secret -n demo my-release-minio +``` + +## Create a Milvus Database + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +``` + +```bash +kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/distributed.yaml +kubectl get milvus -n demo milvus-cluster -w +``` + +## Verify Milvus Database + +```bash +kubectl get milvus -n demo +kubectl describe milvus -n demo milvus-cluster +``` + +When `status.phase` becomes `Ready`, the Milvus deployment is ready to serve vector search traffic. + +## Cleaning up + +```bash +kubectl delete milvus -n demo milvus-cluster +kubectl delete ns demo +``` \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/rbac.md b/docs/guides/milvus/quickstart/rbac.md new file mode 100644 index 000000000..bf0c4d925 --- /dev/null +++ b/docs/guides/milvus/quickstart/rbac.md @@ -0,0 +1,178 @@ +--- +title: RBAC for Milvus +menu: + docs_{{ .version }}: + identifier: milvus-rbac-quickstart + name: RBAC + parent: milvus-quickstart + weight: 15 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# RBAC Permissions for Milvus + +When RBAC is enabled in your cluster, KubeDB automatically creates the necessary Role, ServiceAccount, and RoleBinding for each Milvus instance. This tutorial explains what permissions are granted and how to verify them. + +Here is the list of additional permissions required by the PetSet of Milvus: + +| Kubernetes Resource | Resource Names | Permission required | +|---------------------|-------------------|---------------------| +| petsets | `{milvus-name}` | get | +| pods | | list, patch | +| pods/exec | | create | +| milvuses | | get | +| configmaps | `{milvus-name}` | get, update, create | + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +## Create a Milvus Database + +Below is the Milvus object created in this tutorial. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: quick-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: Delete +``` + +Create the above Milvus object: + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/quick-milvus.yaml +milvus.kubedb.com/quick-milvus created +``` + +When this Milvus object is created, KubeDB operator creates Role, ServiceAccount and RoleBinding with the matching Milvus name. + +### Role + +```bash +$ kubectl get role -n demo quick-milvus -o yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: quick-milvus + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: milvuses.kubedb.com + name: quick-milvus + namespace: demo +rules: +- apiGroups: + - apps + resourceNames: + - quick-milvus + resources: + - petsets + verbs: + - get +- apiGroups: + - kubedb.com + resourceNames: + - quick-milvus + resources: + - milvuses + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update +``` + +### ServiceAccount + +```bash +$ kubectl get serviceaccount -n demo quick-milvus -o yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: quick-milvus + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: milvuses.kubedb.com + name: quick-milvus + namespace: demo +``` + +### RoleBinding + +```bash +$ kubectl get rolebinding -n demo quick-milvus -o yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: quick-milvus + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: milvuses.kubedb.com + name: quick-milvus + namespace: demo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: quick-milvus +subjects: +- kind: ServiceAccount + name: quick-milvus + namespace: demo +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/quick-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/quick-milvus + +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/tls/_index.md b/docs/guides/milvus/tls/_index.md new file mode 100644 index 000000000..0a6bca5d2 --- /dev/null +++ b/docs/guides/milvus/tls/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus TLS +menu: + docs_{{ .version }}: + identifier: milvus-tls + name: TLS + parent: milvus-guides + weight: 25 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/tls/overview.md b/docs/guides/milvus/tls/overview.md new file mode 100644 index 000000000..979dff780 --- /dev/null +++ b/docs/guides/milvus/tls/overview.md @@ -0,0 +1,39 @@ +--- +title: Milvus TLS Overview +menu: + docs_{{ .version }}: + identifier: milvus-tls-overview + name: Overview + parent: milvus-tls + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus TLS + +This guide shows the main TLS considerations for a Milvus deployment. + +## Before You Begin + +- Install `cert-manager` in your cluster. +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). + +## Configure TLS + +Milvus TLS setup is managed with cert-manager and KubeDB TLS settings. + +- Use a trusted issuer for server certificates. +- Rotate certificates in a controlled maintenance window. + +## Verify + +```bash +kubectl get milvus -n demo milvus-cluster -o yaml +kubectl get secret -n demo +``` + +## Next Steps + +- Test client connectivity with TLS enabled before routing production traffic. +- Document the certificate rotation process for your team. From bf11c7499b4750e63ec3affdbeebc2455f1fd885 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 1 May 2026 20:00:41 +0600 Subject: [PATCH 2/6] docs(milvus): use latest active catalog version in examples Signed-off-by: Tamal Saha --- docs/guides/milvus/configuration/using-config-file.md | 2 +- docs/guides/milvus/custom-rbac/using-custom-rbac.md | 2 +- docs/guides/milvus/monitoring/using-builtin-prometheus.md | 2 +- docs/guides/milvus/monitoring/using-prometheus-operator.md | 2 +- docs/guides/milvus/private-registry/using-private-registry.md | 4 ++-- docs/guides/milvus/quickstart/rbac.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/guides/milvus/configuration/using-config-file.md b/docs/guides/milvus/configuration/using-config-file.md index b14de52e0..aa2b22226 100644 --- a/docs/guides/milvus/configuration/using-config-file.md +++ b/docs/guides/milvus/configuration/using-config-file.md @@ -80,7 +80,7 @@ metadata: name: custom-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" configuration: secretName: milvus-configuration storageType: Durable diff --git a/docs/guides/milvus/custom-rbac/using-custom-rbac.md b/docs/guides/milvus/custom-rbac/using-custom-rbac.md index 33247b520..ff831929a 100644 --- a/docs/guides/milvus/custom-rbac/using-custom-rbac.md +++ b/docs/guides/milvus/custom-rbac/using-custom-rbac.md @@ -114,7 +114,7 @@ metadata: name: quick-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" storageType: Durable podTemplate: spec: diff --git a/docs/guides/milvus/monitoring/using-builtin-prometheus.md b/docs/guides/milvus/monitoring/using-builtin-prometheus.md index dc5b8dfe3..85f562d2f 100644 --- a/docs/guides/milvus/monitoring/using-builtin-prometheus.md +++ b/docs/guides/milvus/monitoring/using-builtin-prometheus.md @@ -47,7 +47,7 @@ metadata: name: builtin-prom-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/monitoring/using-prometheus-operator.md b/docs/guides/milvus/monitoring/using-prometheus-operator.md index a0726fd45..3ced47c83 100644 --- a/docs/guides/milvus/monitoring/using-prometheus-operator.md +++ b/docs/guides/milvus/monitoring/using-prometheus-operator.md @@ -47,7 +47,7 @@ metadata: name: coreos-prom-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/private-registry/using-private-registry.md b/docs/guides/milvus/private-registry/using-private-registry.md index 3ebbb5ab4..6b3fb25dc 100644 --- a/docs/guides/milvus/private-registry/using-private-registry.md +++ b/docs/guides/milvus/private-registry/using-private-registry.md @@ -68,7 +68,7 @@ metadata: spec: db: image: PRIVATE_REGISTRY/milvus:2.4.0 - version: "2.4.0" + version: "2.6.11" ``` ```bash @@ -85,7 +85,7 @@ metadata: name: pvt-reg-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/quickstart/rbac.md b/docs/guides/milvus/quickstart/rbac.md index bf0c4d925..b3c817de2 100644 --- a/docs/guides/milvus/quickstart/rbac.md +++ b/docs/guides/milvus/quickstart/rbac.md @@ -50,7 +50,7 @@ metadata: name: quick-milvus namespace: demo spec: - version: "2.4.0" + version: "2.6.11" storageType: Durable storage: storageClassName: "standard" From 33fcd7bf6980fd3506c0a1bde34c32419c953c5c Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 1 May 2026 22:25:54 +0600 Subject: [PATCH 3/6] docs(milvus): add required namespace, etcdVersion, and objectStorage fields Signed-off-by: Tamal Saha --- docs/guides/milvus/README.md | 1 + docs/guides/milvus/concepts/catalog.md | 1 + docs/guides/milvus/configuration/using-config-file.md | 3 +++ docs/guides/milvus/custom-rbac/using-custom-rbac.md | 3 +++ docs/guides/milvus/monitoring/using-builtin-prometheus.md | 3 +++ .../guides/milvus/monitoring/using-prometheus-operator.md | 3 +++ .../milvus/private-registry/using-private-registry.md | 8 ++++++-- docs/guides/milvus/quickstart/rbac.md | 3 +++ 8 files changed, 23 insertions(+), 2 deletions(-) diff --git a/docs/guides/milvus/README.md b/docs/guides/milvus/README.md index ef8652c30..1a94c30eb 100644 --- a/docs/guides/milvus/README.md +++ b/docs/guides/milvus/README.md @@ -36,6 +36,7 @@ apiVersion: kubedb.com/v1alpha2 kind: Milvus metadata: name: milvus-cluster + namespace: demo spec: version: "2.6.11" objectStorage: diff --git a/docs/guides/milvus/concepts/catalog.md b/docs/guides/milvus/concepts/catalog.md index 0df3f06b5..077691bcc 100644 --- a/docs/guides/milvus/concepts/catalog.md +++ b/docs/guides/milvus/concepts/catalog.md @@ -31,6 +31,7 @@ spec: version: "2.6.11" db: image: "kubedb/milvus:2.6.11" + etcdVersion: v3.5.21 deprecated: false ``` diff --git a/docs/guides/milvus/configuration/using-config-file.md b/docs/guides/milvus/configuration/using-config-file.md index aa2b22226..ac4888141 100644 --- a/docs/guides/milvus/configuration/using-config-file.md +++ b/docs/guides/milvus/configuration/using-config-file.md @@ -81,6 +81,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio configuration: secretName: milvus-configuration storageType: Durable diff --git a/docs/guides/milvus/custom-rbac/using-custom-rbac.md b/docs/guides/milvus/custom-rbac/using-custom-rbac.md index ff831929a..718628af9 100644 --- a/docs/guides/milvus/custom-rbac/using-custom-rbac.md +++ b/docs/guides/milvus/custom-rbac/using-custom-rbac.md @@ -115,6 +115,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio storageType: Durable podTemplate: spec: diff --git a/docs/guides/milvus/monitoring/using-builtin-prometheus.md b/docs/guides/milvus/monitoring/using-builtin-prometheus.md index 85f562d2f..1b405cf87 100644 --- a/docs/guides/milvus/monitoring/using-builtin-prometheus.md +++ b/docs/guides/milvus/monitoring/using-builtin-prometheus.md @@ -48,6 +48,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/monitoring/using-prometheus-operator.md b/docs/guides/milvus/monitoring/using-prometheus-operator.md index 3ced47c83..7eac16d3b 100644 --- a/docs/guides/milvus/monitoring/using-prometheus-operator.md +++ b/docs/guides/milvus/monitoring/using-prometheus-operator.md @@ -48,6 +48,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/private-registry/using-private-registry.md b/docs/guides/milvus/private-registry/using-private-registry.md index 6b3fb25dc..ab2852652 100644 --- a/docs/guides/milvus/private-registry/using-private-registry.md +++ b/docs/guides/milvus/private-registry/using-private-registry.md @@ -64,10 +64,11 @@ Create a MilvusVersion CRD specifying images from your private registry. Replace apiVersion: catalog.kubedb.com/v1alpha1 kind: MilvusVersion metadata: - name: "2.4.0" + name: "2.6.11" spec: db: - image: PRIVATE_REGISTRY/milvus:2.4.0 + image: PRIVATE_REGISTRY/milvus:2.6.11 + etcdVersion: v3.5.21 version: "2.6.11" ``` @@ -86,6 +87,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio storageType: Durable storage: storageClassName: "standard" diff --git a/docs/guides/milvus/quickstart/rbac.md b/docs/guides/milvus/quickstart/rbac.md index b3c817de2..b4f50577e 100644 --- a/docs/guides/milvus/quickstart/rbac.md +++ b/docs/guides/milvus/quickstart/rbac.md @@ -51,6 +51,9 @@ metadata: namespace: demo spec: version: "2.6.11" + objectStorage: + configSecret: + name: my-release-minio storageType: Durable storage: storageClassName: "standard" From 38c69399a92820ba4c03b1bff3eb29c8ecf21578 Mon Sep 17 00:00:00 2001 From: urmi Date: Tue, 5 May 2026 15:52:46 +0600 Subject: [PATCH 4/6] docs added for concepts, configuration, monitoring and custom-rbac Signed-off-by: urmi --- docs/guides/milvus/concepts/milvus.md | 533 +++++++++++++++++++- docs/guides/milvus/concepts/opsrequest.md | 42 -- docs/guides/milvus/quickstart/quickstart.md | 85 +++- docs/images/milvus/milvus-lifecycle.png | Bin 0 -> 88897 bytes 4 files changed, 587 insertions(+), 73 deletions(-) delete mode 100644 docs/guides/milvus/concepts/opsrequest.md create mode 100644 docs/images/milvus/milvus-lifecycle.png diff --git a/docs/guides/milvus/concepts/milvus.md b/docs/guides/milvus/concepts/milvus.md index b79d3f564..06bc2fe8a 100644 --- a/docs/guides/milvus/concepts/milvus.md +++ b/docs/guides/milvus/concepts/milvus.md @@ -16,42 +16,533 @@ section_menu_id: guides ## What is Milvus -`Milvus` is a KubeDB `CustomResourceDefinition` used to deploy and manage Milvus vector databases. +`Milvus` is a KubeDB `CustomResourceDefinition` used to deploy and manage Milvus vector databases. You only need to describe the desired database configuration in a `Milvus`object, and the KubeDB operator will create Kubernetes objects in the desired state for you. ## Milvus Spec +As with all other Kubernetes objects, a Milvus needs `apiVersion`, `kind`, and `metadata` fields. It also needs a `.spec` section. Below is an example of Milvus object. + ```yaml apiVersion: kubedb.com/v1alpha2 kind: Milvus metadata: name: milvus-cluster - namespace: demo + namespace: kubedb spec: version: "2.6.11" objectStorage: configSecret: name: "my-release-minio" + metaStorage: + externallyManaged: true + endpoints: + - http://etcdcluster-sample-0.etcdcluster-sample.default.svc.cluster.local:2379 + - http://etcdcluster-sample-1.etcdcluster-sample.default.svc.cluster.local:2379 + - http://etcdcluster-sample-2.etcdcluster-sample.default.svc.cluster.local:2379 + disableSecurity: false + authSecret: + name: "milvus-auth" + externallyManaged: true + configuration: + secretName: my-release-user-config + inline: + milvus.yaml: | + log: + level: info + file: + maxAge: 30 topology: mode: Distributed distributed: mixcoord: replicas: 2 - storageType: Durable - storage: - accessModes: - - ReadWriteOnce - storageClassName: local-path - resources: - requests: - storage: 10Gi -``` - -### Key fields - -- `spec.version` points to a `MilvusVersion`. -- `spec.objectStorage` is required for object data. -- `spec.topology.mode` supports `Standalone` or `Distributed`. -- `spec.topology.distributed` configures distributed roles. -- `spec.metaStorage` can configure external or managed etcd. -- `spec.storageType` and `spec.storage` define persistent data storage. -- `spec.authSecret`, `spec.configuration`, `spec.monitor`, and `spec.serviceTemplates` are optional controls. \ No newline at end of file + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 600m + memory: 2Gi + + datanode: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 600m + memory: 1Gi + limits: + cpu: 700m + memory: 3Gi + + proxy: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 600m + memory: 4Gi + querynode: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 800m + memory: 3Gi + limits: + cpu: 900m + memory: 4Gi + streamingnode: + replicas: 3 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 600m + memory: 2Gi + limits: + cpu: 700m + memory: 2Gi + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 2Gi + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 9091 + resources: + limits: + memory: 512Mi + requests: + cpu: 600m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + serviceMonitor: + interval: 10s + labels: + release: prometheus + tls: + issuerRef: + name: milvus-issuer + kind: Issuer + apiGroup: "cert-manager.io" + external: + mode: mTLS + internal: + mode: TLS + deletionPolicy: WipeOut + healthChecker: + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 2 + disableWriteCheck: false +``` + +### spec.version + +`spec.version` is a required field specifying the name of the [MilvusVersion](/docs/guides/milvus/concepts/milvusversion.md) crd where the docker images are specified. Currently, when you install KubeDB, it creates the following `Milvus` resources, + +- `2.6.7` +- `2.6.9` +- `2.6.11` + +### spec.objectStorage + +`spec.objectStorage` is a required field that specifies the object storage backend used by Milvus. Milvus depends on external object storage (such as MinIO or any S3-compatible service) to store its data. + +The configuration is provided via a Kubernetes `Secret` referenced in this field: + +```yaml +objectStorage: + configSecret: + name: my-release-minio +``` +The referenced secret must contain the following keys: + +- address – endpoint of the object storage service +- accesskey – username for authentication +- secretkey – password for authentication + +All values must be base64-encoded. + +In this setup, MinIO is deployed separately (for example, via Helm) and acts as a dependency for Milvus. KubeDB does not manage MinIO directly; it only uses the credentials provided through the secret to connect to the object storage. + +### spec.metaStorage + +`spec.metaStorage` defines how the metadata store (etcd) used by Milvus is configured. Milvus relies on etcd to manage cluster metadata and coordination. The etcd operator must be installed and running in the user cluster. + +- `externallyManaged` indicates whether the etcd cluster is managed outside of KubeDB. + - If true, users must provide the etcd endpoints. + - If false, KubeDB will create and manage an EtcdCluster resource. + +- `endpoints` is the list of etcd client endpoints and required when externallyManaged: true. +- `size (optional)` is the number of etcd nodes to provision and used only when externallyManaged: false. +- `storageType (optional)` defines storage behavior (e.g., durable or ephemeral). +- `storage (optional)` specifies the PersistentVolume configuration for etcd when managed by KubeDB. + +### spec.authSecret + +`spec.authSecret` is an optional field that points to a Secret used to hold credentials for `milvus` root user. If not set, KubeDB operator creates a new Secret `{milvus-object-name}-auth` for storing the password for `root` user for each Milvus object. + +We can use this field in 3 mode. +1. Using an external secret. In this case, You need to create an auth secret first with required fields, then specify the secret name when creating the Milvus object using `spec.authSecret.name` & set `spec.authSecret.externallyManaged` to true. +```yaml +authSecret: + name: + externallyManaged: true +``` + +2. Specifying the secret name only. In this case, You need to specify the secret name when creating the Milvus object using `spec.authSecret.name`. `externallyManaged` is by default false. +```yaml +authSecret: + name: +``` + +3. Let KubeDB do everything for you. In this case, no work for you. + +AuthSecret contains a `user` key and a `password` key which contains the `username` and `password` respectively for Milvus `root` user. + +Example: + +```bash +$ kubectl create secret generic milvus-auth -n demo \ +--from-literal=username=root \ +--from-literal=password=6q8u_2jMOW-OOZXk +secret "milvus-auth" created +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: milvus-auth + namespace: kubedb +type: kubernetes.io/basic-auth +stringData: + username: "root" + password: "Milvus" +``` + +Secrets provided by users are not managed by KubeDB, and therefore, won't be modified or garbage collected by the KubeDB operator (version 0.13.0 and higher). + +### spec.configuration +`spec.configuration` is an optional field that specifies custom configuration for Milvus cluster. It has the following fields: +- `configuration.secretName` is an optional field that specifies the name of the secret that holds custom configuration files for Milvus cluster. +- `configuration.inline` is an optional field that allows you to provide custom configuration directly in the Milvus object. + +```yaml +configuration: + secretName: my-release-user-config + inline: + milvus.yaml: | + log: + level: info + file: + maxAge: 30 +``` + +### spec.topology + +`spec.topology` defines the deployment topology for Milvus. It supports both **Standalone** and **Distributed** (cluster) modes. + +### spec.topology.mode + +Specifies the deployment mode of Milvus. + +- **`Standalone`**: Runs Milvus as a single-node deployment. All components run inside a single pod. +- **`Distributed`**: Runs Milvus as a multi-component distributed cluster. + +### spec.topology.standalone + +```yaml +topology: + mode: Standalone +``` +When `mode: Standalone` is used: +- All Milvus components run in a single unified deployment. +- No separate component configuration (like datanode, proxy, etc.) is required. +- KubeDB manages all internal services automatically + +### spec.topology.distributed + +`distributed` contains the configuration for all Milvus components in distributed mode. + +#### spec.topology.mixcoord + +`mixcoord` is responsible for coordinating metadata and internal cluster orchestration. + +- **replicas**: Number of mixcoord pods (default: `1`) +- **podTemplate**: Custom resource requests/limits and pod-level configuration + + +#### spec.topology.distributed.datanode + +`datanode` handles data ingestion and persistence into storage. + +- **replicas**: Number of datanode pods (default: `1`) +- **podTemplate**: Resource configuration for each datanode pod + +#### spec.topology.distributed.proxy + +`proxy` is the entry point for client requests. + +- **replicas**: Number of proxy pods (default: `1`) +- **podTemplate**: Resource configuration for proxy pods + +#### spec.topology.distributed.querynode + +`querynode` executes search and query operations on vector data. + +- **replicas**: Number of querynode pods (default: `1`) +- **podTemplate**: Resource configuration for query execution workload + + +#### spec.topology.distributed.streamingnode + +`streamingnode` handles real-time streaming ingestion. + +- **replicas**: Number of streamingnode pods (default: `1`) +- **podTemplate**: Resource configuration for streaming workloads + +Additional storage configuration for `streamingnode`: + +- **storageType**: Defines storage behavior (`Durable` or `Ephemeral`) +- **storage**: PVC specification used when `storageType` is `Durable` + +### spec..podTemplate + +KubeDB allows providing a template for database pod through `spec..podTemplate`. KubeDB operator will pass the information provided in `spec..podTemplate` to the PetSet created for Milvus cluster. + +KubeDB accept following fields to set in `spec..podTemplate:` + +- metadata: + - annotations (pod's annotation) + - labels (pod's labels) +- controller: + - annotations (petset's annotation) + - labels (petset's labels) +- spec: + - containers + - volumes + - podPlacementPolicy + - initContainers + - containers + - imagePullSecrets + - nodeSelector + - serviceAccountName + - schedulerName + - tolerations + - priorityClassName + - priority + - securityContext + +You can check out the full list [here](https://github.com/kmodules/offshoot-api/blob/master/api/v2/types.go#L26C1-L279C1). +Uses of some field of `spec..podTemplate` is described below, + +#### spec..podTemplate.spec.tolerations + +The `spec.podTemplate.spec.tolerations` is an optional field. This can be used to specify the pod's tolerations. + +#### spec..podTemplate.spec.volumes + +The `spec..podTemplate..volumes` is an optional field. This can be used to provide the list of volumes that can be mounted by containers belonging to the pod. + +#### spec..podTemplate.spec.podPlacementPolicy + +`spec.podTemplate.spec.podPlacementPolicy` is an optional field. This can be used to provide the reference of the `podPlacementPolicy`. `name` of the podPlacementPolicy is referred under this attribute. This will be used by our Petset controller to place the db pods throughout the region, zone & nodes according to the policy. It utilizes kubernetes affinity & podTopologySpreadContraints feature to do so. +```yaml +spec: + podPlacementPolicy: + name: default +``` + +#### spec..podTemplate.spec.nodeSelector + +`spec..podTemplate.spec.nodeSelector` is an optional field that specifies a map of key-value pairs. For the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). To learn more, see [here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) . + +#### spec..podTemplate.spec.containers + +The `spec..podTemplate.spec.containers` can be used to provide the list containers and their configurations for to the database pod. some of the fields are described below, + +##### spec..podTemplate.spec.containers[].name +The `spec..podTemplate.spec.containers[].name` field used to specify the name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + +##### spec..podTemplate.spec.containers[].args +`spec..podTemplate.spec.containers[].args` is an optional field. This can be used to provide additional arguments to database installation. + +##### spec..podTemplate.spec.containers[].env + +`spec..podTemplate.spec.containers[].env` is an optional field that specifies the environment variables to pass to the Redis containers. + +##### spec..podTemplate.spec.containers[].resources + +`spec..podTemplate.spec.containers[].resources` is an optional field. This can be used to request compute resources required by containers of the database pods. To learn more, visit [here](http://kubernetes.io/docs/user-guide/compute-resources/). + +### spec.serviceTemplates + +You can also provide template for the services created by KubeDB operator for Milvus cluster through `spec.serviceTemplates`. This will allow you to set the type and other properties of the services. + +KubeDB allows following fields to set in `spec.serviceTemplates`: +- `alias` represents the identifier of the service. It has the following possible value: + - `stats` for is used for the `exporter` service identification. + +Milvus comes with one primary services used for client access (Standalone or Proxy in Distributed mode) and four component services for distributed mode (`mixcoord`, `datanode`, `querynode` and `streamingnode`). There are two options for providing serviceTemplates: +- To provide `serviceTemplates` for a specific service, the `serviceTemplates.ports.port` should be equal to the port of that service and `serviceTemplate` will be used for that particular service only. +- However, to provide a common `serviceTemplates`, `serviceTemplates.ports.port` should be empty. + +- metadata: + - labels + - annotations +- spec: + - type + - ports + - clusterIP + - externalIPs + - loadBalancerIP + - loadBalancerSourceRanges + - externalTrafficPolicy + - healthCheckNodePort + - sessionAffinityConfig + +See [here](https://github.com/kmodules/offshoot-api/blob/kubernetes-1.21.1/api/v1/types.go#L237) to understand these fields in detail. + +### spec.monitor + +Milvus managed by KubeDB can be monitored with Prometheus operator out-of-the-box. To learn more, +- [Monitor Apache Milvus with Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md) + +### spec.tls + +`spec.tls` defines the TLS configuration for securing Milvus communication. The KubeDB operator uses [cert-manager](https://cert-manager.io/) to issue and manage certificates. Currently, only **PKCS#8 encoded certificates** are supported. + +TLS in Milvus can be configured for: +- **External traffic** (client → Milvus) +- **Internal traffic** (inter-component communication) + +--- + +### Example + +```yaml +spec: + tls: + issuerRef: + apiGroup: cert-manager.io + kind: Issuer + name: milvus-issuer + external: + mode: mTLS + internal: + mode: TLS +``` +The `spec.tls` contains the following fields: + +- `tls.issuerRef` - is an `optional` field that references to the `Issuer` or `ClusterIssuer` custom resource object of [cert-manager](https://cert-manager.io/docs/concepts/issuer/). It is used to generate the necessary certificate secrets for Milvus. If the `issuerRef` is not specified, the operator creates a self-signed CA and also creates necessary certificate (valid: 365 days) secrets using that CA. + - `apiGroup` - is the group name of the resource that is being referenced. Currently, the only supported value is `cert-manager.io`. + - `kind` - is the type of resource that is being referenced. The supported values are `Issuer` and `ClusterIssuer`. + - `name` - is the name of the resource ( `Issuer` or `ClusterIssuer` ) that is being referenced. + + +- `tls.external` - `external` controls TLS for client-facing traffic (gRPC / REST). + - `TLS` - requires only the server certificate to encrypt communication between client and Milvus proxy. + - `mTLS` - requires both server and client certificates to enable mutual authentication between client and Milvus proxy. + + +- `tls.internal` - `internal` enables TLS for inter-component communication within the Milvus cluster. + - `TLS` - uses only server-side certificates to encrypt communication between internal Milvus components (e.g., proxy, querynode, datanode). + - `mTLS` - not supported for internal communication; internal traffic is generally secured using one-way TLS only. + + +- `tls.certificates` - is an `optional` field that specifies a list of certificate configurations used to configure the certificates. It has the following fields: + - `alias` - represents the identifier of the certificate. It has the following possible value: + - `server` - is used for the server certificate configuration. + - `client` - is used for the client certificate configuration. + + + - `secretName` - ( `string` | `"-alias-cert"` ) - specifies the k8s secret name that holds the certificates. + + + - `subject` - specifies an `X.509` distinguished name (DN). It has the following configurable fields: + - `organizations` ( `[]string` | `nil` ) - is a list of organization names. + - `organizationalUnits` ( `[]string` | `nil` ) - is a list of organization unit names. + - `countries` ( `[]string` | `nil` ) - is a list of country names (ie. Country Codes). + - `localities` ( `[]string` | `nil` ) - is a list of locality names. + - `provinces` ( `[]string` | `nil` ) - is a list of province names. + - `streetAddresses` ( `[]string` | `nil` ) - is a list of street addresses. + - `postalCodes` ( `[]string` | `nil` ) - is a list of postal codes. + - `serialNumber` ( `string` | `""` ) is a serial number. + + For more details, visit [here](https://golang.org/pkg/crypto/x509/pkix/#Name). + + + - `duration` ( `string` | `""` ) - is the period during which the certificate is valid. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300m"`, `"1.5h"` or `"20h45m"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + - `renewBefore` ( `string` | `""` ) - is a specifiable time before expiration duration. + - `dnsNames` ( `[]string` | `nil` ) - is a list of subject alt names. + - `ipAddresses` ( `[]string` | `nil` ) - is a list of IP addresses. + - `uris` ( `[]string` | `nil` ) - is a list of URI Subject Alternative Names. + - `emailAddresses` ( `[]string` | `nil` ) - is a list of email Subject Alternative Names. + +### spec.deletionPolicy + +`deletionPolicy` gives flexibility whether to `nullify`(reject) the delete operation of `Milvus` crd or which resources KubeDB should keep or delete when you delete `Milvus` crd. KubeDB provides following four deletion policies: + +- DoNotTerminate +- WipeOut +- Halt +- Delete + +When `deletionPolicy` is `DoNotTerminate`, KubeDB takes advantage of `ValidationWebhook` feature in Kubernetes 1.9.0 or later clusters to implement `DoNotTerminate` feature. If admission webhook is enabled, `DoNotTerminate` prevents users from deleting the database as long as the `spec.deletionPolicy` is set to `DoNotTerminate`. + +> For more details you can visit [here](https://appscode.com/blog/post/deletion-policy/) + +### spec.healthChecker +It defines the attributes for the health checker. +- `spec.healthChecker.periodSeconds` specifies how often to perform the health check. +- `spec.healthChecker.timeoutSeconds` specifies the number of seconds after which the probe times out. +- `spec.healthChecker.failureThreshold` specifies minimum consecutive failures for the healthChecker to be considered failed. +- `spec.healthChecker.disableWriteCheck` specifies whether to disable the writeCheck or not. + +Know details about KubeDB Health checking from this [blog post](https://appscode.com/blog/post/kubedb-health-checker/). + +## Next Steps + +- Learn how to use KubeDB to run Apache Milvus cluster [here](/docs/guides/milvus/README.md). +- Deploy [dedicated topology cluster](/docs/guides/milvus/clustering/guide/index.md) for Apache Milvus +- Monitor your Milvus cluster with KubeDB using [`out-of-the-box` Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md). +- Detail concepts of [MilvusVersion object](/docs/guides/milvus/concepts/milvusversion.md). + +[//]: # (- Learn to use KubeDB managed Milvus objects using [CLIs](/docs/guides/milvus/cli/cli.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). \ No newline at end of file diff --git a/docs/guides/milvus/concepts/opsrequest.md b/docs/guides/milvus/concepts/opsrequest.md deleted file mode 100644 index a83fa19f8..000000000 --- a/docs/guides/milvus/concepts/opsrequest.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: MilvusOpsRequest CRD -menu: - docs_{{ .version }}: - identifier: milvus-opsrequest-concepts - name: MilvusOpsRequest - parent: milvus-concepts-milvus - weight: 25 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# MilvusOpsRequest - -## What is MilvusOpsRequest - -`MilvusOpsRequest` is the operations CRD KubeDB uses for day-2 lifecycle changes of Milvus databases when supported in a release. - -## Current support status - -Based on [new_db.md](/new_db.md), no Milvus operation types are currently listed. - -## Expected CRD shape - -```yaml -apiVersion: ops.kubedb.com/v1alpha1 -kind: MilvusOpsRequest -metadata: - name: milvus-ops-sample - namespace: demo -spec: - type: Restart - databaseRef: - name: milvus-cluster -``` - -## Next Steps - -- Track [Milvus ops overview](/docs/guides/milvus/ops-request/overview.md) for support updates. -- Use [Milvus TLS guide](/docs/guides/milvus/tls/overview.md) and [Milvus monitoring guide](/docs/guides/milvus/monitoring/overview.md) for day-2 operations. \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/quickstart.md b/docs/guides/milvus/quickstart/quickstart.md index 4a7c4480c..9078abdab 100644 --- a/docs/guides/milvus/quickstart/quickstart.md +++ b/docs/guides/milvus/quickstart/quickstart.md @@ -12,37 +12,102 @@ section_menu_id: guides > New to KubeDB? Please start [here](/docs/README.md). -# Running Milvus +# Milvus QuickStart This tutorial shows how to run a Milvus database with KubeDB. +

+  lifecycle +

+ > Note: YAML files used in this tutorial are stored in [docs/examples/milvus/quickstart](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus/quickstart). ## Before You Begin -- Prepare a Kubernetes cluster and `kubectl`. -- Install KubeDB from [/docs/setup/README.md](/docs/setup/README.md). -- This tutorial uses `docs/examples/milvus/quickstart/distributed.yaml` as the working example manifest. -- Create namespace: +At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md) and make sure to include the flags `--set global.featureGates.Milvus=true` to ensure **Milvus CRD**. + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. ```bash -kubectl create ns demo +$ kubectl create namespace demo +namespace/demo created + +$ kubectl get namespace +NAME STATUS AGE +demo Active 9s ``` ## Check Available MilvusVersion +When you install the KubeDB operator, it registers a CRD named [MilvusVersion](/docs/guides/milvus/concepts/catalog.md). The installation process comes with a set of tested MilvusVersion objects. Let's check available MilvusVersions by, + ```bash -kubectl get milvusversions +$ kubectl get milvusversions +NAME VERSION DB_IMAGE DEPRECATED AGE +2.6.11 2.6.11 ghcr.io/appscode-images/milvus:2.6.11 6d2h +2.6.7 2.6.7 ghcr.io/appscode-images/milvus:2.6.7 6d2h +2.6.9 2.6.9 ghcr.io/appscode-images/milvus:2.6.9 6d2h ``` -## Check Object Storage Secret +Notice the `DEPRECATED` column. Here, `true` means that this MilvusVersion is deprecated for the current KubeDB version. KubeDB will not work for deprecated MilvusVersion. You can also use the short from `mvversion` to check available MilvusVersions. + +In this tutorial, we will use `2.6.11` MilvusVersion CR to create a Milvus cluster. -Milvus requires an object storage backend for metadata and data files. Make sure the secret referenced by the example manifest exists before creating the database. +## Get External Dependencies Ready + +### Object Storage + +One of the external dependency of Milvus is object storage where the segments are stored. It is a storage mechanism that Milvus does not provide. **S3-compatible storage** (like **Minio**) are generally convenient options for object storage. + +In this tutorial, we will run a `minio-server` as object storage in our local `kind` cluster using `minio-operator` and create a bucket named `milvus` in it, which the deployed milvus database will use. ```bash -kubectl get secret -n demo my-release-minio + +$ helm repo add minio https://operator.min.io/ +$ helm repo update minio +$ helm upgrade --install --namespace "minio-operator" --create-namespace "minio-operator" minio/operator --set operator.replicaCount=1 + +$ helm upgrade --install --namespace "demo" --create-namespace milvus-minio minio/tenant \ +--set tenant.pools[0].servers=1 \ +--set tenant.pools[0].volumesPerServer=1 \ +--set tenant.pools[0].size=1Gi \ +--set tenant.certificate.requestAutoCert=false \ +--set tenant.buckets[0].name="milvus" \ +--set tenant.pools[0].name="default" + ``` +Now we need to create a `Secret` named `my-release-minio`. It contains the necessary connection information using which the milvus database will connect to the object storage. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-release-minio + namespace: demo +stringData: + milvus.storage.type: "s3" + milvus.storage.bucket: "milvus" + milvus.storage.baseKey: "milvus/segments" + milvus.s3.accessKey: "minio" + milvus.s3.secretKey: "minio123" + milvus.s3.protocol: "http" + milvus.s3.enablePathStyleAccess: "true" + milvus.s3.endpoint.signingRegion: "us-east-1" + milvus.s3.endpoint.url: "http://myminio-hl.demo.svc.cluster.local:9000/" +``` + +Let’s create the `deep-storage-config` Secret shown above: + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/deep-storage-config.yaml +secret/deep-storage-config created +``` + +You can also use options like **Amazon S3**, **Google Cloud Storage**, **Azure Blob Storage** or **HDFS** and create a connection information `Secret` like this, and you are good to go. + ## Create a Milvus Database ```yaml diff --git a/docs/images/milvus/milvus-lifecycle.png b/docs/images/milvus/milvus-lifecycle.png new file mode 100644 index 0000000000000000000000000000000000000000..45aff469d4e7ddf5c72230b288b1b2a180fa1695 GIT binary patch literal 88897 zcmcfpcT`hf6fTMeK@<^DP-)Ve(yR2|yA+;hgcxLfF}RmAU4c^PAtCi^vz6Nv?b*HjT5Kv{88B7(W{pOe?4pN z+@9XNWBSGXfbHpTf5IE68+ZK7WN;LkA3rOUCFC3sG#hG|fKIFir+cGwS%+2lj98MG zW{IPrNg}H~N&Gw6B2B(CM+X(v)vS2HK-T2MX~;?t4nZ@O4hcPJVg><{7Dtv;&mAEm z?kgU%hd$yfH-OuGZqwm_KoCua7#=brnuE!cI&FJ zAq4dM`}+h8KqoB1HB$0`8uI*72NN`gbBZtscnL^Ns{3mtTuCju*fwh&g z)$04iRNsKc^CN%urKOKIk(*l+%{RJKbXo@s7JVG^8hKR-v&CmL z_%4mE-F=E^QJE!J1V?flD?1lgLa?v9w)be3{!1?p4`m~xq?+yYX^iG9l47LRjJJro zgPfSKgE_TXP7^TG2GUcAOYYknM*rMb2}4s7 zi&?rE``Xgoy*z289X}l}2Gb(hb&K=&XavuPrz$HeS4wR;=$kBtxhzhszZ8{tf9P1p z>Dq^fgm&oButt0f(=;WGOLi*>STI{gUzV$7=hIOZAOj}vhnMukty72KtqY&GE-bvf)sA}iy^r!6v3~3UOQwi;7 zokyL)riN^^BGr0qYJteJ-@#Ww&p0`^8#V3|fIvKw_oV^JM(LvNmlPDJ{46YF=H~97 zM%`szVhy%~x|~-T-4kfDa!#9#noU%;@(s6Vjra!$w(mmQ^^JB7nwr_|piC#pbY6&f z^Y=F*A|f3lBi~jBKn%^}@*qqHeZS@E0?!V+zCF{;{{Fpr+Ic$)z8&ls7Z+?B7yZKY z?Wq0DA4(*Q7i5`GcRyzOI;zDX3iZyjS$&3#0Gk5!l6#4t1S6&^?4ar?&x_BhjC!HV zArFD}TcWW7c`^{9$^G1CgqNmrVG!TztCQ==p-akg)h*-Jo&C-VN{g~d-dmR>|NKbH znx1ieGG0W_R6pzrHZ!Gn&wX-_fr^T%4p-0H+YM+jiU_N{j3oS{PuCF%wL?)kn+avY>9E9IoGBW_#z0VQ1ChqEsyuD?fh5I+&7doSH= zdKS&!3Hu%6v??vOY{#de6 ze<{o1=bAcFSavGmhV)w)KP$;dn7uegA?tdJCPdS%nMiowW45^4wT6*fb@<->D2pVm z%1@~JxD;_k4 zj`Hj0*&Q;VwI!D8VJa8Ww9LOVI3`aB)F3!Ha-oJaHQxW01P#xK4I4i?fgzE z?`$8@NwAMzvdMtOXXV<>t;MaZyk)~!DYqwB#V5}}784Q@TJ2;ea-1HS=}2_3TDQ&c zn(U3!max)i{^|_peC>OAC~CwfG^_UkKU9XalaB_Nmb5cN2(Y`!L#NG3X8l^6CrY_l z+CGXFdcLJM6@Wnz%6+$w`G|>ot;Dunje=JNx(I!&5im1}+Du#!h_0H)@E)La?}*g{(aoESWe&kTJzQpqI}``^b?OX0Su&KMZmU^GYW;*a zkbFHTk1ijA$+`MeOUe-C&l+ycgg`fp4g__hZ<4)o7%`R$3Ory~DKYF&0PEa}17&a84cI6fSb1sD};og}TGnm1+jzS{2vDi5V1tbI$B z(t`!9f~sC~U^L*`YLWj~q}^y`uiDso#u~=Xy%BugN`01v{tDJ(cJcTaT+^y=8MKLJ zn;3q$>3wn@LMrVXt*Z2t>*VC*Ed%BMvh?yce$Gx#f(ZYVAL#Dr2ny0tsi2IGC{nvB z`Wh9v&VrYcNjmFLA1SkzO8Ck9_A#ylvFZ2D~V_! zu`tt;t@y*gx4+93PH23cb=__x$HtNXO7cHe*>dX`+}}vzD*L7*{)Q^W;7 zoat*X@b~B4cpn1HjPB^#z|#-sYVae1XYa=*qhn5VHN^z4j{A#sv{y8!B#lqUrwf8k zW@t^Uv%it~rh)iG9%y)aR<^$3#jY4Vc3(0w#6&!N_%Kn$z%^Z&I0j~kSgXRq|CcEAzdc%Q^a{>o~x z1dI4pkhp~nb#QS~yH32wNR42j{E7j2ZiLs?B(q$+Fxc^~s-b&TYv)U88o^%f;LKO`kGO5Kiro`FbkJi^>wL+BAj9(qm?hqQi); z)0I;4xu9SYs_j;C7TQ=&??w;>?_`=N4Oz{FRN?YD^z%{%Rm;#Ag*Qq~SjJLTw8hoe zPR(#dvlrUooW8MD5@k!~2;QHCh4N0+vWnbWy1Y2UZoN><)}!mN+Z`Vqd>s^IZPVIw zoT?nPW%$W>KA(OGi5Tm&a7kw<$;e<8o&KSz>He=myk_;3(&(b{8K1Enk3HR)hk=k| z+OsFX+-ofHn#2H`*Zk_Go?han;HDg3frnK`aD_lvlnDLOddH*0FJ9$i8|oGYHI@Nk z?6gRGwt%v?a>|nb4t0{mealGW+p(j1e+TRKzSq`Hno8{(E31A{+|d%w*enoq{8vMw zwDVL`I*oR>bJFQ`WGhMT4Zi=@8yBo>b~<<4Xx8Q+CvNYT{e zc+NV;p-GB_cLv<6AzInv6D3JdT({|>kW@Fv_Z(&r?#j5XQZ`S0$GyZW>z5ieTIn8# zdq1FGdWA72rSlq1h`@gS>^7A;Ggf9(Et`6;P{{6CMp0~gK3pzyGT*{kwi5RXPtLV~ z?Lak#RN(QkgpN_H=mSkzudOfLg<=eDt8&E)?nCW}UU-69KUd5bE%xV|*8Z`{3$j|? zH?34uB0apkFuO&O_F8&h z*vJ|hdF=VYxJlsob5(2WoV$vFfekflTZ?M(9P4|A00yo8gIakxX)2ID5i62z!_F_9{1+%!gB1wS9iHQiUIVH0=)N zFsV{UHC+X{&*mhUvjOwBq`p?Ea!bkfH(jNqwFtueRG%@txGkkCVhRz&q(k^eD*PpTP z+I1RJuwlxl_H+4hU#ot&0y9F!fvcvfa^T|s2IO)@br2YVZC(p({7>jDg~dNw_2s7b zp0|~mN(fi2Gf7Z+%P-e@8zfEvK@=ZJMYnc~JP{3d)45wgVIe=VSkOan^1|5eevR$m z4;1{7f$3|Mw>Mnh_~bUA59tF~_wwl^psS!{lf`%WETbr%IB-AYi4?a5!{B_ef#bG% z=b!dOrZQptY}P~=sgym=juD*)Kup_3i0%y(4fV0JE4UlJE$SpEFO>GQy&-DI zUZL1@#Fg5h#N3HXErDwr6ig2uCM=2;Mu)HWND*EOU@LMO{pOXoQi?Zv- z*&|;+4U3Xbk`BTtw zB=U`i4h0dplR|d*XLthlPEg~1pIHBSz`>|Nif4w;0)e~#-a;G8s}XBK$OsH!Oi%zH zPl6#|#sUhv_~1Ve!Zl8rO?_`906makZ{2t1t!npPNqnY<#bhwY&qRRdPVj@n$C4X( zf=P`mIc6mmGZZn1AYqLg@52bJaNmazINt&w(PXi4iyRTXpwGy6?NC`Fifuf8)tz#j zMNfnoB5;=4dwS3e=>wRcO@g`~o!MMu36QIHx*DMPM$;~O%27=aS$s&-)pw2Z--1-KE35MNf-Z{cvP<6Vr$=l#L?2K|+Al9U z)G@dr>(tVAOQju;6B80_qKDswgH;P3@)O+kOuCGRTTx*4YuL3{XP^OpL2L?|mBPEb zQP*xeeeSNE%2l(ra}K&i?S07>I2`<3SvVO|FDd)77$h-3 zlCby|f9d^I(i@Zm5OL5K7e53r!+$u5TyCqCZT{W3`sW{gCnd--PYZQ6HNT8+ii9`B zWL@om&9#ltgFpL%Zd*MN&YWGle6}`NtF#(ivUMB&XjXzwW81ZW>qz9aeF?j=a4h8e zzVY*b;M(;)n>Z!rPrdLs=^x0ngTpS#bSB6XZ#HepS@01)j1irlY znc&IhQG4sAnsT~}Pu(-(aFk3d0hL^zL!!|l^jutPXFjW?5ZdQR1b2{1*o_(`Qve-uM3O=k47Ot1%J=z_zp)sMm(ucQjz)7k-i2E@r?chJ=EX&u z^RwjAe43}($HXEnpHyry3;KOT#coA?uZ!DYr29(>e+uxdRc)T=dQYJkBl-m1owloO zQ-9d~9`}|yGbTE6qr&KYGkFe9cC>Fk>94g6Pq-!ed))ebpMn$ z==jAuR&C62eqexxRMXauS&!B{UFR0fq;?wIyUd`;fJm?&R!l_ z=~7cfb|IXIJp>b~A3~3(imMm$5s#<`YqBzfQ0{E6kb6mwPIVD8_x30Re z@rAyr4ok^4xe0x?rfB)*EA<&aSvMW(>zPd@h5;Pmt0$$j$-uENv^LKw*!>@=wg;g z5Wu?+V^UY;vxk03v4D}{Z3NU3B;FsCiD;8NFnw)Z6Am_xx9vJU_UwNtIyEkKoOu*y zN+ZiVr`Gu;g#o>*QK@5+w8cLIfPNOl>mkhcEd6rLh7H}G5jFqk~bS)#$hU3{c zZc}5fl9lmn0dp-`%6p2-=cI(o$=23CfJMyKH8pjtH7CVR&`DhuKzuyWb|E%(Zydho zUCt&Pb+Ugb+W#HJMnp?oDS8&a6GPTyVDncoZO^#mdZq!`*ob`95t5f_{`&QM z+mG}9&Xep-SN?hG1&1oG@B5-T8CAbfhGvEVz&}B#kGNA#Xf*jEq9g4r^bcwdLjqc2 zVQ(|ucj%C<_txLPvaILBQdjlpR=AAsagF9gtq_~J)^sE@|K`RMcO6+G0zo>C&i;h= zs=`-oq~=N5i%?xKCuCoXW2jh^ETbT3#9qUNt3Mu?PzVk->P@8cc)o7+$SRY4ZdmH( zEdUAqD713>-EK3oy>1wHqWQf*iXVlM;_B2~I04`~ zJw3h7BBOGvrxgpXn`6a%!I$SWXMi$>>|u=&2-och>5Ty#^a9~VDlPSe*S&n+IOk{ zhL)Sl;%=s|&w_a*_LPI})Y*l9B{=*l$^`nk5pW zO8b|Jo6{b{8f6~qIyHOkRG2{X7q+!`VMgU|yFNc+@9C$MJv5l`xi~<0-vR?c3C5;2 zW4DvEvC#CpP}9JJ(V^*Sj@5L3XYbnu)p2PzP1u5(Wk?bmC!p zZoW;(Va}1hJ$18&vCKJ*>1`iI)$d9*0q?I`+nk%sNT95QiWyhk>=-djv~Y|Azc8== zx7ERXZ}k6zY5*v2yU!;nuIZ|wsQxM_Nzk{9Lu{SNeYQ;q$;9nA!4T{PJGeB}Y4R>9 z@>^#vc*+&e?7m7<8qDoD!@)^SErDWZXNx6W>4na}X;BK~K9DfIUhr3QSWODo_Vum7 zL=wn9qwhXJjJu8xshoMFu7+x=UjInZ7oG9hX<(`FU%n*;xBxFpRx(xW z^l{XPy-nH^<)c8qx@zl6lf$CPNz{vejXE)hYMbq{Z4}B8F$^0p{y0A6=6!KcvT|Bh z-Y;3NVIv{7vb5*ElE~K6Xxlu#h4-m8RbOnweKPPj>XsDbQG%&kt7!G_pA4f@`b9iA zE6Bb}Q}_9gd@ryn-2%qqMvktx%-Q<4HHHLN3q2!An!KDAy}{R{4>$tKAXZl`2?CwSo6nUc2=`zfytQqpI>mEC=WZtED$J z4aZHa7ih8BMA;^lLp)_fF^Wb9&>gVBpKtpsxPBoQq0f_ zZ&{r=l05l$)a25xety{@#bt5X#@%ya{45bx=Vo!ao$9WRa4M!5fehe>c%@``rnBoB zx8+X_h3IvBA91qd*_d%=6I)rd(+nA(;=35!-rUds6NTfu*G87ItG(`@a|Of#Ziv%^ z&66Lu$r*3cy(fs8djM+ZB07PYSE&azz5PnCGt>CyW5H?Ne8Ud4UP#{S%xL0*Kbd`r z#qYDcGRK_E{l<)N)0s`G^UjW&b+x)~vSv6|4U(BqhzzU_av|a+Gt51xE+lAyouhaU zuo;aacp)6Bh++e--_qe<^d-~(m4ZFJEiQ{lrD3Y;G-(A_zt?_PfqbtnSVFdF5PPZa z_+=ENl9LTL(zc@|Ro z2^+GPSuwRg92pqwbG+CE)tTiD&Aq@CFJ`D%RM|Kj)-WEXE$$mZAw(ZU!CZa{oF%56 z{>@SAt<*H;huqh(D3=7MMc8Zu9?+0c&|d^FcJf%81i8s5k{PMb?sYj&jEiba?OUyF z_dBm3-I#Eyn^0|Qs`<$=d^6lH13%gLAfGh6kH<0iNw*=IxDsJbH{p-Mz-QJXG-VXAbxCWzzeK(wsRIiuFdgCcVlMKF5*_ zznqFb$#;y%7a$+8u4PR!s{f5x9ZQ0FqMs0PCj>2!BKZSds&VofS-Cc}hY2h*JDDCTHaC7yQ zHUWkR2oML-6a+Xt^b>zlk^pcT4#o##Cu>OK9Et0Pk=C9hCGnMe-#lZJ3Bz4)(1V)i-qGMjNqGDLuqU~qF|YSd!%_Eo zUWW6}wRZMm9}feG!?#c8=Zl`)wsm4&`uVaDo8G{`&iN#AZYIZ3E{29f>hl0i4DBGt z5TBUW|34kkwOc%a)$6p^=E}=n3eqGv^&K!uX7n8T045NmtboP0e;GE%x(pH5{vTFk zRyQ^+X>;n~yZh&sHNYfc;MQ-9sskV>mQwO3fT%Ezs|ScSKyz4x91>Ln@K;_O+}uJ( zoBd&!sVZY#9f(4Oh_({$si5%DcdPLX1!Y!?zb^|kICmTC-`5OkcLybAEP<*6jCa94 zDSQKqF-}J;j&1F{9c7h)*0Qg$N``>^7I#2}j$Ehb;;VKS;ETIoEr9nBN!vug17VkE zMq*vv!tr*E{U}?_+y_F;G)bwjDW^3=nKhj64lSpdK7!)uHmdkfjye8cSk@*OZ#%Zp zc95Yq5kR+AAsDL+T6#z=rXwgIVD-??em8&97O{Je7?VQh~U0zbgPVFX)25dLrv-njgdG9F_)p+LYE|G|usnsyc;K7(*e{5tZ ztvN{NdayByZpJ0<6hxQ~2VTV)nA?4j4OH()tVs7+h(sN5n4!}BQM?FIuN=v@b+!(O zT>v+hQ%_l&WT+KVBF%a|eH(zS?v(1cQD9fdv;z>8<*6`lfEFUoj|GtHD+X*d0d#eL z7oa|c;-V=p?SAFG{$8x(Fo&PBM^mll)UxvP+{jpEmBLfeq z)#BrL(<}g-79|nl{T_z?-2DyC>_x2n7wcAE! zA#gOo^P)wjVb-g^3OYLOJhc(4#>ES_75ouRa55K4oP35P9a%$rPrE94zsR?g-q`;9 z`3^rGv?S9vK**Exo(5t#?k>A{N^ySKA_*ZrVm;eIxYY_hf3N;oVjgv%x zRLq3~djCE6dLAm1%*J*vv%;gRZtMYU<7Rq@qM6V9N95)>*O0NuI!(E@T#rYlm8cho z5;ft_(A4u?K+4U{*tmj@bwX7sCo?N&K8xCg*Hff8bp$p^NuKUKYGVQ?`pnOpE z_3@a~CqgqmSw;pnt>(YqWhU(z*98I!{_55Be4)`&mp-I;_RCpfR(e&UI&~Ix9k2C7 zd80JnWIck224(B3VrWeYFiycE2(iRJh>HsZOjgVaL+Jq!pLFPl!k3<| zq`r-$brm4ucCI;hiPV?bFpQR);GX)L63mK2=Z&?cL6{({oxV6qy*T>GQ%y(U&+urj z|6>q+j}T~{M|l%bf+^nk&W+uaz^DHag<(GapTzlC^Qvs_XP0rlVNlX}d6DLJd9uHx z;gZY-WhKK528692k+E|DcP@evaQ=RQPB=I1TH{rRl(_?!O}dEM%mwNu?mdR!i_PyN zhz|e2^P+O}QW&Y<6)6|FdZ$YeX{?05-)y+&j4w`-i0%@hh2e62dVY--?#OkjH%k~s_K+Y6PG~E!|lyA&zi!Lpq5`B%PG&D$KSoEa(41Ml$zc_x3&4Y4XJpR zy_?TAKa!b6&|V!YtOgc@|eeaYT<;(%ABz_>%Ofr|9uwnIYGg;U*h@Kr7< zMHEm1Xz=f|3Em@s&ll0rH(KJM@1v-U#$^pq5^ zTp!AuOw>=yJ?xqZ7H*)S8AXCP>7yo)yb>`|?xp~qMeAE?a(_*njfNSId%xkgZ!CNv z6UEv4%SeOR-P`M^hQF|_^O<}9b;Snj2(a>u8h0K;4XA~7Y;5>y?i!8;T9_eSNx(|k zm7YxmjDR_`ZA%L`ckzX*cf|`jFi?C1$HkfJsuVm*B^Ljs(oRA|TheeiNdGDI2wDBd z6o_Go?tP+^td%%jdyZ)zgpKak;5O9+vT7*wc)sRJ zrP(i8S0@&tZgYZWUU_5J?9(e491G_T4kchI&QY`JoUVymDOGRJ0&G#D^7S~T-QdH> zc~cpgf{~FPXl!hkMEs!h*ezmbpp~2{9Sw#jf9QdTqN>Ome(K7XI=cGuEvk@%sh`!PCUpG!DfJ$AYzKsm)&Ti zHoo_k?-Bbj3~}=&&G3OSuDICcyzLOL;O3@DbpGR1iMqcv-|~v@NWA+lcN)vpIeKS| zy(D0uCOX$l3H(V+-a~9PwFW8Wke?4X_Za+u+TBa{A4~p%4`w%LY^ewpac9F26cSiY zJUjDGrY2VwvJko`%mQTH=s2mbcDBk_lYUlLBFk1Wkn#PbR^nTf70t4OBjTPdQx!anGIL+{Ufvx_8eh#b=Bg? zf)3pWNU5k?3{#|}W_jaL0phL!dRtncqmloG{qnVjRe;C*~NfSS7 ze(~?et{w2zNq>a+iPe=^N#({K^-gC(sWKQ|TrJ5BG($dhO~o@h3fUL?jcjzNul)w2 zes+~A7SahuJn3cY^xgj%ofwsy%`~vsSmRy@=s0NoI%R=_avo_q(Ri8#L#QRN;KEBa zJHFP>eaNz^=t(oN<8l_ASGRu(gdidRuzR9Cguh=s91NCt9Pc#s=7Ou_PZmC;#tOZ} zaMFA$J~8rjk<(Q2DK%MHW1|t@UJt*&)zu}VO}DcHAKI%}t)3IH6Sf7kV>pvlRIsC>)invQDE|RRF z)-!Afiz_mW^9d2aE-&BQ!QmE7AW&Mcgoe+KZyJJEQ~3>8w7BBA1J_-E%og5_j|=-m zsDAFW3%_~UrX<6aqVjX}Txd)X;$+*U#clLuBf+reSu1H?xBpDJa`>=8lb>+F2y6`1 zZP0Ml>yqyL(_%FE>d0gXm-d*NbP$+d_I0t8^vh9oQkv&NJ8m~}+fk~@L>U5(6)z#8 zNj*D3)tO2!a^{z_Rrqu2>roMO{JO?BPo`$Wm*i;KJY-7(CaO5;sh`qwy_ zFB|8FIYW%*L{o4%sp50t*TZk!2fwS=sW}; zQj1}4K+;-K>Ol1>kMv|*Ees``By}ZFP>^Kf>`Y|K|efy zgBJ<{!ZxNt8p1pBGbepLvI^UcNQ)Pdj1#B>lPZD3sbatV6-9S59X{7kb6`?4SV*1y zAQDd_$I?Q;XCy>Bv~y^vYekZX#ilO+`tZ?9mEaG5^eOJ7y?VPoblIXY6=*U#@{kZ; z(MGag&jLCMERVV#mL*Wu*42G_c1R7;NYPK>LOgopd4d1g%PmRz`_A6u#k~jt!R>gt zPaAc(eLonQ)YDCwLieJ*mr`I8{)hj}=JVmU23!37smwqr3^xFc=(gR?j%X=r4@U!i z>B4KB(nj*Qr>ZKmkwGRV`TYl{ghZ7;ORKxWf&K@iu(;@<#reWIdPwa|QEkC(XO82YA@BN-`3n~ZqeRN z`Pl`)$IyV&;o!LFXhzmj->uS9M%h2bCMnj!cfF>{Z444OH#QpOCT-H~ApOBXqYtLE zjzo)p{E%+y2trM(p6r2VvBdQg?5G4VfX8%ZKm8+shMXY>o5ad;EoyJk6lr>jfWvw^ zkr5p&_ckp$x*c^U;1h6EN0z)z{Ha^DB-Jy1mVo?>RtZhkbi4bd!J{rgM7=gpdr?pU z|EEta@&vzp+Y+onlUL);u3l+t@qR(Nx?~6AR?Dy{zwgA8+XgM7IgbLGPDk-4>W*Hz z585k>5qwS27p|&8CFJIs0XP8PW;G-Do9Vkh=kzbd4p(|z`VtJxBU4~wfrDMPH2xfH zt7!(STm1nii?l*2T*I(+)^y>~^Pzl5h;@fYKGc)7JUgJ9A4BwR=dhhbS zR8qKOv zIsY8tWuf)8DzCX4-?(zwUW@@}w-YgR&8v`#&3~SVG>&TIp%<%R>N=hSJ?lTUK3n8u zE#NvQD~|xjtI>FQP_CZyx~fIVy6%n5LUj`nP{oi?2=kL`g4;h@xFz|^FkF_^ZE!JW z-Ac@u_w+q62X9C9YU4vFBnzqmL|7Km*>g)KtwNEVM$m`Sad~R2SliBi0Q%;L%!HQGY?XM$_I`&s_V8Cd>S(LCZ!@2i=TCS>xIUOK=(!5Ev9mcFc|}0tH{E zqNlTOMcdkf_Mc25X+S*Z!eE?snj6ZmZYJGEuew$G;o)HzC=v3Q z)7VP0d#uEwKavFW>P7h3ED*3XO^1b3KAr(m#T1-{ujej~W?fL_n|5R+U#0OdelE&M{CF<(eCiI!=&$cxohZ^%1ywCCK8vFi&wrM%uq=)ZxUyfLUiI9wIYlc`1KPN3fu{x_d z4j;0ylH!62q6t*5vve?UstW`*WF!`UfkbTRpbg_2@@jSZ?AtWO35``5wJ`i>O3F#6 ze{Eb)*q16So2C38r!~N)K;5KWNa&O^FKu^Zn}f{M#Z}Bf{@hNC5uI6lwZ3ngv(#`j zGbKT8%A$BJbXtlANzrzO58f8@TV5ULV5B`&5hh^)|Ely#lwR zo!B|z0KwJ;Z1UBnhkvd{dYOQKcFwF_uJs@&#cSE09r)t&eGvji>$7U-;-Ymsgg~WG z@Sm04#p%J_SYi)9)8D_md?RD5M9=iT%eM+a(H( zNS8E0hv$8w=f{DRL$goY|L2^+cZ;Lh;9y14azjy;A21BY#+r)nT7~+>0MA+@-zT}Q|pzlK0l%{g+{j!3BF7ez& z3G8(`YOH;;;t&iQ=~l{-BE{VN-^IHTVqq27e8~V6z@qV}i*JradLP*D)&Gn28H0_= z%*!Vyw+;_6fZ7u%SJ4KmZqXf#8j)e*W$t-BUN>VwJmir{{JjcnY)r1*&qyrl4N3CS zSb$m|Ooapw6AkOTe;0e;g_>2Gm?53WRNvfIGOlmqMm8$-FwN{Db&|ddP&@6m>VE#_ za|=NbIa@!>Y{AL)zLrDbafZKUfYeE(kM~Y|>rFD{^YeXXlqoI<;}k&6-+j`W2ag(d zRO-J=|J7V0ui24QQBT20Ei@-st+nU6>!3tCIim%!JQc=^{QsJ!B8!vK?Zex;X0hZdWSnOjcyvNP@8b~IXY2RJmKjY9xh zgI>z)oUCi*EPj&uHqP5|8-U-i0q2Q9p+@b`lK<%UgoM=ov0FR<29$Y&1T?^|tFfCh ztK0z&lE+l7xy9JZn6pQfa-z=%qBlHgyM{eiJs_Na5we`w0pe%F$LtK$iFyxJI;O;`fP&~#Q{Z00&6BMG;7evFu&h# z^B3Tzd2-ju3jS~h8{LSyMF0}{15g9P^+(f;=}i|JvVc7Wl@;30%gQ*5RJ;xakz4$* zNn_?+-J!bpAb6P6e;f4b-wQxaJRS>40|y9z;BEHFgbw*M9ZBp72oVC`Vy3HpU|C8p|B)DUVyFC5_766WRFeTm@TqES78h&DQfNn z6f1pbPePds8FHC;+SP=jf`={ZIcF(+*E}MS{-&0)qyJ38emNX60>U!Z$!Intg2A-&>JTg2*2MhJxq)I_>;k zEwDO3a{?h&j~zke*C&fWAp|l_J_kJa0C8NOw+80<{~d2(N&oh!F!w!=i)j*#&i44Q zt$QpkTHGW_23s*hAsrm7Fl!7f9hAi;I~7~g^aeMXJ_k>cCG0dlj`>Q>@KJJ=xEA(} z_c0j(Gf7$g2s?uNc4zDzAgBQb&}@s%Z8ek6Eqx|&w9(>fvDkr zsAawo3qgk%h^A$5`?7J+20eHD+uUC>8ncl&edx$=v#R5QB1jfE@{4EL6WrR3>8 z0VV&*5NPVIt4sj`FLG*<6rDGWwpP^cmoLqgwsJ1e;!0VD(mFDljxkb!kWuK|EAZq|5b3!6NGy!V|a{GqwMXh=4gU+HC znIxdiA^<z*Qa5rrx&<*LW;4@%Nmgr1Qe9B12 zsb3b&XLUBdFijWcpB2x2KLf66U21K@TPEDha?2x?4mz%hsGlG>sj6h|;j>Xc>}DPg z)L9`ri6_OIRBavEoPX@;Hb6#6TK0RIJs3&GfAkSLk>BHCt`?cmIEU$u08mMQQyVn> z#m|;&{GuaZWY;zLDSfaM`>Z(Z1bHN+uw2)-OwPKLVmueq9n!h=2CV)J-E9<1*oEo~ zTX(xCxQR0{Opy=*HN3o#=DRtfs!UBagy_c;j5J3L&+!d*po3Pqd;Aktx_;=y&}Jms zung?^;0xhf%lb(gFf7+rpA9+P0!ES*dVRJdf{0ctNWhjQK?PtjT9htU3`6VX=EDLk zE%hFY%v;)r zLA)H5kwr5vXVW25&y}o$dAQm%%Z_*41Kd88U)T`0FXtWb+JRehU<2&5X*&t)3h7n@ ztLL-X;x|sAhgm|3rzb_ zS}WgtmR$NI1Lu-{B{A2Fd@E{ojAUN=H1!4z6~bFOx03`kySWovRwErfu)Dpn+QL`K zV^mySz)A0&@y&+C)79YV#>H}Az}KS9c8z5#^kH@YI4n)fgDCy1Rf#(L!vgl*&d02sMV$XS)mING1ZnI%hgyY1WfoV*6l$w= zw;k&{C-y!|p9YRNFXf5;v9)eyEIjOB8WL1of@`i(xK#};;gtw+YKDJZ?&D+btNMxd zbCxTlVn+**Ys_4>vulgMrdhwFzSys+UWSv}tim(2;LtC*e#_bRms448TAL zqX13)0g>;}{iqbWoqQdz>8ajRil?>lDHdBXE4Dk!;96KelxI=)C+?px(&v6m{eri% zpwE)K-lzt+p0ox8Px6vT^aZE!#_&7TyCG$p0pX;JS@!OHNEsBrvdkJ>Q&3oY>iMiR}D$qvWC3`^7HU0FURDxZSCaLnGxbilbYA)=#JNO<{|`drZ$68=xQa{9b%9@3a+X7az5?P*Ha9@rd@@XHAQ7U%nk zHcdD6?rH8LnoIt&Zi2g(dsZ(OrBjW!No|C3#*D5Gd%OOjV*!@|<6%+lcrieKH z!Ypz7(t7#+u%XZrd?9~pMpo&r}FAqjEk;klSDo>I` z!_h{KsLu*@;JVOH8F|Ah6cT1ag7-(iM0=k6okA}+@Ykc;Rl(|B(Yi72pi2K^GK98#z7{LCsJv1sFw0Tw7t9Op|H1**vaH`R><^B3`A)r zmYe^x1WU!)SfO@->y4h@q)*cIq36R|Ewy>6PV76j@i#@!_O5cz)bk6>vMn4n6Mwm$ zs5eb;*EtZXroP}~TX=G(=JI32xwDPBmlZ9;LWNE+S z=2Ci&Z|tmKWmQ6xA4VC7%3lF^bBPWZ=SA1|-PR`j<$RJ7buXQ+;ne$X2Szt@Q}&e7 z?xPztZ`b%)m{PV7h}8D5v%d7zFmUQc>f6y(ORzBGu>MNj*`yH~OnALU?iQ<>rr$0x zZ6yplVwHQLhT979df-i@N3mhdHRhQiY0VRVSA6?$>(bOe%EEaNILbok?bW(06)h^x zUj5j>+`Ymg$e1>^OdMR~q|%w(o|pt#@m)R5wZzSRqZhcF1o^h&yPVhOFvf|uNRzZ2 z2EDTV4Bye}!Vm{h*Z)uBNe)hGIK-IXBvCJ~p?uoMb=a*COblNaH0Ph07hjd!E-OZULkP>v5LYQO$L@Ci zV<_2LkANq;-%(dyrIJ3)g_jsE?YNql1&++0Rm5uJ{tZq;F@+BkbLV}58IHS~1#Y;c z-d}6%kKWAR-0YtXK*oTLPTd)nYd9Nb8)0{BNZ~eTJ^gwAfV5}xk-fCqS0M8ox;Uc$>#5L#;V5u z#oK#DHT8W_qbdrbA}T7y9(o`l zAks^M5FkJZe23rv8+VNRcwg>$N#2sZ&n|1tHRoJ9rVAb?V>O(S8QH->o>|AMzuIgX zoks!0;oMj5|C~j-F9WdyKbvm$rjMYX=c9y}N<6LOWu8_MU7>$H!nQzc8*MUV_!3Ak z&*lG*nt08FQPglLU|@-k^5iTe`O1U1ef9}F|5r3|_Vxcyn&N*r!P$@ge*ss|xI{lb zOP}Rwz{7CqKZ@Vk*Z=oF&olC!gEgM156-3l7$R-@LorE7`?gK@Gt_eiVCd;tS_(C< zPnx-9ewXS`VWaZIV;V-s$Hx~C042?&)+Qo{m-27zj0w3M7bTa4`~&N_5FJ8Z4y}ij)|7TEv) zA@@S*IhzyJG#=fL5EmuAGk07MTfN>@vN_=s-q3~FVl z6)=NW1Yn`lv(MWQs|Y}MT}Otynl(B$W@}5tw=AbQF9Jwok~bL+=uU`r>=s0dP z&T1q8lu{kwodVB7HuXDGcN^m6N{jLDE;*yNw5AS)*ohy*)E}Aq=AuYT`w(~h?w<$Y0dCEg`RVdYF=<{;x;Dfdkf2ZY8TT>4h)_k-UgXkO3oUMGD}!k z`hKW&mo~_#-`=XhPK=N9Lkd0el^{DL9 zbk`Pj!Pd=Mf*a!zvzrphAY0c`b-nSXe}{Y(sTiR{^52KEOOnXBFiQG zlcvawr_pgj(02W>s?)!bLTOL9K!UM?491PFnTICTK4f^xfM7btptWQ30UH~Y9`SZ7 zBweOJV^A0%8dV{7@(>O~e}`mtp>7MWTDov=2s4kxhwC@if}HOrx;A?=MX@u$ApRZf zl4cL3nY#nerE0LGg@qDafBIwh>wM>mIe2`3l+ZS`#;%GSQ9C}bByCJx^v6P9PD;K} z$Z4-kH~kVoXkOW0m2hpW{sub}z1z#2d>FOed&@D_T@YQ2kbp&vmvwt@daZ?b5|U3g zg9fyby2I!Kq!>u5o?Rcuh=pL;Re{nASH&o7GFxBa-qFl{!}N{~E-UEXE*yu_vN&T7 z1B4v*K@PF}VMR^UngAVAOtPjkM@I7%AwwBG7P&0!PjE~2q47+$;a!`$!2jPIc>0b_ zA{&ex);GL6vJk8AnRe~HhSmMuAi691;os|!+@N!%6GcZ&%Z~wXxYe87>y#F48HMoK z$!wfdBrMqFX=%Tse7?7tf1nt$nKT=qT9EEv<{^0%Ub*b$+doNF(J20C3_+6k?HleV zp*U^>8%z^l&ckKvvqaLZc}qbKN;lT`eL#>B%sKptL;hr#6-haHWKUmN0Vj!zstql5 z>Kevx(C9m^mZI{%byGgzqY>;N3z8&B4M`**@7cab?(HV=7hP zWsN|-c8CA(G%6^T3L6v`YjpR;>J;XtwH|0z@4*8C)f%H#Y~WO3mXF6R97KSfc4@@aG%n(?_XXkrKmUmDU-V4^6w|%tp(>H&Wj)E!r$S&0Wn3$Mf{d7>RpybbEu7c;^Lmx~W}23nyQ@1{W+R10(OpBKLyQ z(^@-VtKWkDxph#W#JaGg{ZNT1k6SLbd2-!-L9JHNr;2jhzZjMoBMHfL#^K5eop^tmY@vLK^>7?R4#Tn(vB{z z+yM#}y}@#~X!#}lo_1?035k1Y9ywNjS=-XGB?PWD_2gx_R-&tNFq~okS{Pf@tLRWe z=ykiF+4E0bTdc67hAcNQ-C%`_%wG%n+b8xmU6j{m2gW5;SUSj|Z;{U9VHI2seAquO zpWZsrR)!=A%3bgNP>xi+o!{5m=sXeyGT9y6OE`?px*^Io zxFM@hEx^iRv$qbtkq9XM*#LIF_sO@??RUFMwAA^xF25@*snxRs>Y1aMsYf($m)M=W zJob$Xp-Zl+H+eT7msKT|VN6puW$#A+ez;iQ*(RGkuy%wk#yN(DXLOwZLL@qd-ZQL! zd9Fu}K%~rHuM11d+=XkKy&)uUV#M@TB8}PcIA~$lE_uaMd1y5A|;7+?jgQjM{ zjMPot9;}5WC;FsABvs#ttI+LT=!*~~sV1NDMaNfb!%kc}ZNfL;Kee#hPg#@xmg?Jn zfh(y7sJ?I|gUe_3&^i$>GU4kpB!d2xrFGjv5nH`aw`K4RxI;bP)8#=um3vWKc>))H z{3`H^Sf(kRju2pFc_d>WK6>GaFEmcwBH{6%$*!NbnKHh&H1$hZDo{n!=<$|%6)e^> zenE~-RNQ%F11Lnck)>2HO@~Itp<)5&aJ>xz#0e^hrrTZ~-E$$rbhOxKZ<-M3@+M-+ z2hwOw@^S#fOWPp>1lSn;n_z6COQW*@3sfOGh;!4qH$kwRa;k4%&wLF>V|K~0KVyI3xRa$l7 zfJx;!*vP&;RT`AHXMbY6r=CG4<_hmWN7rvXr!zjU><}GFC!`Z-Z$HChzFiH+Sfe0B znf`KclTwvwvn70sR=tMUh;re*M$*oDg4b~3KQt@({z2ZK^ zd@zL4Pz@XWXoYzvHZ`WGHy22mhQ>~ISGXnNmz|OC#=A@0sszEsPZ|rW;IE6uur1Id zZyBy{mAA()Vux}djoGxm!^K#Sj_Y*N`p7a?b>=9{juni4X7?U;y39Mt8K%3rIqT|g zzRMUI^mNw;Lw^yhc;Oyu=8inuPy}?X-cb*bTQ*l$%5gW!5A)3(gVGP$*-Oj!rRMVu zw;zm^G*#N)S#)UlX$?KKH*6&_M z7?o)d7e^Yb52y137BhKQ`T<8nK~>%AI8R=+XsJ#VLOcCI2miHVKF0Q3U{SA*MHtO+ zCE2lJ#zyX5{`N7~j@Q+6Z-ubEp7qV_YVA>9xLIdJU*I;`N^a9BtZl-N*M3uThIsS&JS zz0qRox?zFVp?9MagN5P?_z19jY6>TSz8UqT`dgo?G6;}Op1eRMvOuBp z#Aah`?Y>zSy*V7{g+GY8c75-ntk9oWu>GNJBT*9hI2#($PY!mC=-ozdCiRyjt7zjownq7Q9 zlgkBisvge0W*~X5D>l=ZfPT9&=8h_lb+iy;b`dfjRM)#&=iMRBKSCA{7|r;t1K%vF zR=PUrGL%>Sx~_bLnQHXRK*f4UQ7mTspU&KS#043w8jU_IxwoZ7E13M06&3F+;)hE` zsNQ2g5{O0!NuTgc6fx*qZNGK&LOf6d%9K6wzL(yndtH$AI|+R3u2U=ia>ncFlg8sR z6GC8lUMzCH4ld4{}{oG{bU%=3V5S3yc z{-r14CytsP4(}USh>AbX<0eiy%Y^I{W=r-1UYiLf@f1D1RW@&yYpV5hDvRSU{dSf} z$^8+?+o=*I2F=y-r)^q-0^yNg5}}HX9dR?+4%ilep;@9v?}i`1*9)BM*H@8ue@lj z=;7}Yd66m=uG5nQdYx*Bk4J&@__q8sOH04JX_a)l>*ySR+$I~+<00sCA?iC(UN=ip zx9!9GX3F1gox@`}?VXk2FdM>~j@!$A8Q<+!Axi1`vPm)MjDIT!Qs)n|HT#z~(RX~q z=gEyOH!YvJB-y@uAMpYH*C2H{RO9MKrPNQB0#Hd+3Ho1kohA6(1I|ctwA5L z8)fz6Z{dEhMZVyVP2ND2y%DqmrGQ)qIu z;){tx-@j^i@xi_Ao>o`cM?|OeMPh&d;?Kf|dSesRBx&ns2F(jBz$pyW`a}Sel^(hN zvaDa!nHzoi(MeLk{z@%8YBrKtWXpDrVNgIAS&C2?8{HBHBaUc*5^~p-1tff2;o4qu3oiFxKkUToCuKIRbB= zaK2~d_>U3S{r=)PtC+l+lmnQwna}gTzke^kXvIj4M173EBX8N#XaeU@>uD>MH+kCS zrc>`#%9{=Ks^BB4Ykx8d+Ov7(3d_+JlqpxM3ifRts3>Rf5Zr#H2JJX1yG?yTYbYkv zh2@%zb#zCCmXr9V$HmashrO1jOUtLu@lo#2?X>oLGkz-SG$<4@0Kzd1Y9jyXlwjj= zi$PVOY7N7=MD@Ld$9nVT*BuY)z~KqW2@lHG>I3gdg`X{pBN46(J*`^^Y{nA-ON4pY z^A76~h=57$i~qzOJkXIUh5O8A7q76Mkzgd4Z=QE;VdhJ7u3 z*;`lV=pG1teV=47+KI2e$RI#_IHPBqORmA!S-Y8Nn(_3QG5BiV>FgX2N03=W;7(?t z$RKS{S%zl+35A>H`i*HPuT`PZ^}3F=47o9=!+LI%4gq9_$6 zc{;<-^1FqW9@6T+7EM|`xy$&@W~4&a532d*u4SF-P_Q!M_&|R`&(!?UheKka;~0-l zi`RST`3Z?fg@t;;l|x@e{l|oumz+Bs<7?kWk5Nb#?sPK^2r%4bXU;afqZ2~)hid+rCbdaEoPbG4^QEm7%ccN-Nga}q|g1p0a|J2W}VI4 z!R=F+V`Cth9wO2)Bs5_wPh^j>6D7`Nx`Y zzuk@EiDHb(UY>$;dc%i&LxFX2rNSIA{i+q^#yk_c=&TuyHt1H+Ve--fblX20oTh8)8)^pu(DHWP}F#efnf+Q-|E%(Eda0V3$G z`?o-_yp|kwG(Xy-kEFd>&)Hf=%Dk)Hl*j(*Bh3bdV?mI~9Vg z7TRY8l*)RwUDu7|`^3M$5P&t^Of)R9>d7W+&fk&n4j=rhcM_kyJA~RR%NYk>))$wA z#Q=aUp?M)n=9cNDBDVt=35TX!aM4?noxR_rnMtrn9b=ySSOTz=IaYn6GRbyCqVo<)d0&6>) zF4m$OZy$qBSsIj|{6)X2twvj6!RzwNwc{vJo~$2#e~&H)8Zv<5sX4t4 z^^4P<5{^;$B#<`kHk}t|v8c0V+7<5V#uZ}9@1xZD?uCj43(aJmrvCU0ns&n09y4z{u;IP=U2|oR5ZBNKsxmQOYk8Cgw;uWK8 zBTvWVbLFgBV4&`>=SSRlui|_fZk)gWkInO=lA{uypxGj_zojMJEW0;J9&2ODk}?a= zjZ50Ye6JB|srJvo2d#IapZ3WYaa?al0f_M=+wPvq+*6QBbSa_SqW)%U=u0||M2V+3 zI4;=tQR!AZ{;eBH6n9M*%FT$=g|XwJKxm4^Ds=bRI@t7j70NOGUBERTCS0(LR}-N% zCWf~;ssGyb%f|_Kb@{yL<3`7_mAO<^P-h}TZbH3l! z)NS$^Jx|Ahm08$9RleS)Ux+BaUOhE+R}#laJZ7gYF!wKr8$ztk(0mZG8{5>c7292nHW@bTl4V*kHE!p$h2 z#r3Owi}zQO`ohEC9WJ)^6>{o^Rr0&eOb=r!GkG3DoSgINmfyR;f19hAU3_}Wg&xtl z%f2b3x*Az!kD@?^mjp*^58r2TU|p0NGke`_;EZqATL|hNyL%YE=XBio!zO(56pL9+ zDRvBoj$~WWBZhdVz6_pkU{}TI;*wmAa8N7+7u^~zq6L$q)HF|8)rzG{O)WTvDu7hx zL{VP6gv$TehXcY=%cSW>t`?h_z^09@1uW>&xl{M?WF9S+ zQoUb?*GyK~#-XO|h`N1)M;4L*LH1le<^F03g;oJ?$J;@K^hYyY3o`9SO}=h4D0n>5z~8 z+r;0Df%A~YbBrLgd~soEV`gd(?insI9gb~ktuPIqMCT$wBux?$`Wj6cQF76z32DHN zEJ!7+E5a=G_}}dLv|?k~uR?f_uBv7C%QueFr+G%>r!8+daWod9+uNzJC!38gy}PKp zc|~Vay?%o5=5Et$N&8mM!@NrBLWgLN19XJqs)N4s;TKM~`pJtEnKv1j**!!0f`bD} z1NYnSz+8#g4(BAd)1sF=iR1^&nmxPf@`q!|MZ3n!BaES*+uCHCWt(6O?J{D@d>ZUY&5G1ASrqNmh}ellsgMk+yaBp z&6cA;fWx57moB*~JWSrqpii^3@qAJr(11Uw*l4mevP^v`o%Ugh(Cx1!7WsWT0v4_3 zbr)uVbCU@)3m&y~gknomQwGyc{)H=~kjB|{&F~J{mc*QKS?-27!$$Dd7QTI$QmcG^ zHDGI2H67gS)!;3rFhUuJ96?eSt-5Z>oof-@f7_*kdyS8--i;Xb8Nmy@^XkZ5o2 z=T-sBgyOtAGjKEkvSWj;iAMsBvsAAU4WXlVm{BCV`qs!8zW+cp?45EfXh_ zBgsY#YJQpAwl9Ult_nTTA~Dy)S`|u?qQY~KT2wlKTI%=<&4uNotqo}pSOJOW{KH;5xb=r zRW_Q&?A4BO3^lOr8mE}XP1G>!kzrl&PV3ixb{Vi55&|c)%sGS=(9)+TVHs{%;UjA7 zU6xECGeN1k7kppZi}Hr~_;>oYCcV|Va#Jp5K<`EH*qDCPt!m2|KMrDb-6yR^pUm*Q zJWLfg#BjfNske@>Z3bEX|zG1wVShu>pCiK2uDrsJz5HhMG=WSKaoeZZV*!7Pt`oG^~fy7(LROO6@W)rl1!xh(7 z(_v(MBdIm3mEX6eS5j_4y6b$e3rC*Hbv3)eL^k^s;{?pa;m5v92$Zk z1HbQfB~_)piOz*ZX=Rl+!ce>IYN{QNhrv~3{OoC`)?{9_XuO;2ri~ck)g(pfJE*}& z!8;YveZQevDcEmK$Q?T=J~J*jeev6mU;4v)&6agsj|yLQu_!tvuMfT2u{n-N4&9ke z-|wxjDti%4G=A_TZc8`ofl<=IBFP!E7>$B!x^*e~Yvnx9!`->&kh}pK6gD}EpgwCG z(5lOy1X!kW%{$ET?n{5Mp?qlClmh5x4TnSWVFjE*(N*F12cnvd49uKe6`HxYe;+Oc z56Bg!t|~}`KfL!v^R-E;H(|yx>Jr4=`&xql!SPEkQjjk>ZRbAPq*Z_$yRV{sj_Dfj zVo8ha;p(AnoB(q|@Pg!^UdmRomMH_|<`3f-^{A(>Bw{p;K95^M3tVzz3^xm9qMJDRK&D|S7b?MB*jpwaP59WpdVT|c z$QXe#83tF&G>yYiG(W^Gf1gePMr6m`SRbmW(s} zvOLGu zLIPZ7Tq3?h4RkBn8>r??;d)aDYB_>`Hif#U7MV;7dcWPPvoB(MB< zPWKtO=3@dQuWnDnt&-CFt~x*VMyFF_##5dq{Y&{VDj4rSg7i+u**e%!GuyKfjm^1h zKKO8+B9|AR#o%*etQH-A+>B6p*ioXkFR5bNPx!3$0MhvpXoaH@B{wy#{zvpn80N0e z!1TV=k&t}-`Uua5qe87<%a|rQJGHOw54aHLmk)pn<+KfocWf`Iem*{6DAe0T#Zv0$ ztNs3EDuR=0JD#1TMp8my*mbmYI)oeuYAl}cISMpYvq6S#Mzagaq0p}ebqvSQGxwFP z11#2S8kJcvFy9r2_L}B|!BLmtlIl+Os}w151}}TZeQ9yj_xP^e-&N)0v=)k1n1dEB z*lauza~D6bU~00_NU8CuQ1o5O3D~z1xay;;O$d5!R#p1#IZHUq3qOvBZ8jIKhtDK|~m*=tIy22DB1 z$XOg^9N`stJ`_vXP;wmaf1j^nv;SeS`9ciJdNsdj zMXLWn3qR{Z`^B*HM#&~}^pnFQf=>RD(x=6m7hAdwh<rsyfVJ+XY*7lNnw8=ojDGJ(?@tN-k2=oTvd-;qpTqD2azjx(|Ti``U4AL&-Ya% z@sg%jz;o^@!KsIr*ogrnlyrmgrcfQV5-?%Cx-2|s+ zsWzyg=aqz5T}TstlwLTkh_;hHk|y{6a*_0b;Z0d>c>_n5yblrnG_B|f*d#dV$ik-f zR025LvzaY0H$Yok*Ez*2p<5Y_zM-NQoGLqdeqX zl$Sj&IFJ!#C%Xm%xSp8W#ZCHi^_X_ocnT1VpE;Evf_UBb$uEL(%ya`YGh`X3U z%!=Rbrdn z-%E7tJg5{dh*{$ml_YrQ@u(h=sL%3~$01dXFH)}74yGm_==$8LNH_TfIN$2*4j9T_ zPi_o`Zr$t%$dg_MH|T!Jc&{6*n{<2iT_jg3Jh1f__mDaYk&@xBmuv2!l-lNMk;N2l zRNgMR710NaZtM+NlKH)I%r4*ok9~c)RCi0gC*$$?wKQGc`)B7+(F#$**(2Q5KggB; z2L4*lQB&*lr}3=T-$!l5{ZA^Ay@`5FYw`{{`KjMp!;7k%1ALv}xz#8+LcYwCF)y;W zXec)KJbqvp+6v|VaH;sNuOPLsYa-|)J3gJJ>JG8Xc0;-YvUeV(fMIGiptDNVq z`raL#8`vOFbO$G9l<l9sXLo9=ZVm0nb?q!qO?rN)rIZhcfKgPsWDNOLr?vWlbBSp}vzb61Znc_aDaZ4`ZE&>9c|& z^EnJj*NmHW_~ZA)9o6M=-MmHbiF8isM0)?X7%?R~(~Jjh`@+Z`Ubi`+V1Encp3X~( zSd!{eA+}f@@~Ik0VHVM8bqUw1)SB(6;o?RG*{89J;^tQNhgyOfSzW%R_bTs;buadx z8b0O-H0!<&i#o!y&r-B<9P$~2H;Wo%`_Gv6D!>KzypW*hUP+Ajm zyL5!in*YY~AP#)p5F&oYxc3XTacDOy$H z59)slmoLtEJ>37-6}Jk`qgzCrArIvPyFN~Uolj?iTrhrX&z@hmhMco|Y{Jhiep@>e z(p>FR7mNIAIJ8jpW+G78D@scwYQE211VRt1lt}^Gkpz4|zHY{qQ{xm7dpc94l8EA> z-afYH>(nCCAG9nMtylu#LAhryER;QvuGGaKlkI+|4n#f<`R)-?lx4%ov0gsOb-yGw zK|Mf?#O!iV5>1Dc)Hvsv9(7t`--K$voAmRzGut8Qj(=QThcs`$^mTBab4vFs<6m2t z3TUOPOQgnw_4wH@w|(@ONt;ta`uLQm*y|er?s0l$Cyst7n>hxFhDx8esofD)_!HZk z*c-w4m+8$_N2^ktK(ZiTToK?DQE$Ut75f&+SqX59fIgHKGtA;!xbG0lILPxZczAp| zMhEn12h?^5MbA{*$ejF=>0e@V101InW;sGHK^2$)&=$JZTuLoziYuDW(#SOS zY4ZH|VU$~fZqnspmLV%fu!&-TGMVlJtB9<;CV+DfqwbQXO`1e|pM~*vG4*Y*_qD9; zYxA4oUj<;i#swD|b?sWWlM-#+!kQzrSaMr$PkBaLO5Jqq8sC2G!YzMy&8p1wuw?$e z!N(s5g}gH(##&dtL@rtB8=nF4(+!~t^OHHMLZ(l5g)C=yH{k2N&O0@|i3kG2aaO8lyBR|D0R*F+gxyo31Sb+BUsMKFBrAf$oX;Sud*aP1CwVrNK_ zu~!q@ZJ=7f)n}~xj#czU5rT|p!KycPIXt}mH&(k5BdoKGgXFF zJlRfj9qt2F<{LyvPbKE?l3R8{*T^R)7V%nM@9oIY<0}*wAJXx;qM|g3%{vj)R4XJFB8?Q@OjAC6JDEKih-R@} z;xN^^)EKtR{EXo8x$p1vc@qjw`teqopaO=LOM!&A(oEE?9A&x{(H(HejWmHq5p7Y5 z${Ij68Eb`H@4u5~>45efl-Rrnc&IM3!L(n=u&-@QfO9|DNxzO7z3_05hR4$*4zceo zMNcL9jgxen2#D5sM|^%(@@7mdrDnQs^j1M%i}Awuf1#zOC#KCw65Td>sVG_rA1J!% z;sxah=Q#COdF8D>S+t_@*s=KiR}VbiN#LuB9XG!AnQ7gAXX;o&`>VMKGKqzfkT08l*J`ZX>mKkm*jpw-h|zeOTP8UOZ^JLlwvdEHj40OqaX8 z;1TrKk!N24wbT1+1{*eVL(>~$L%nXVsqZNG*9Lx!DA9?>oPHcn@5xceS6pU7Vwx^(7$0(-!0_7SoJE#ea(8AZjDks3X^vQzZ%cRY# zgy0eZK{Mu`wGKMD^}gO}>aSxtTQ?W)j%Bz_FzA%+jF-oL#r!!~6gKD?tkk(|VaNm7 z+xts8yxghUqt@_};Hp1g&wYBHt)z;yM`<@Z=`7bJA1v-Gz|*jqxgXow`N!+Sq}AyF7RmdQGu=+i zha$K{>S6|y+cZNGl|T*q2Maf`nRN4{>4;{*a&SU}clW@@>!z6{prQu;hNwAX!_~yU z&!cbKJ|)7@)2Pv(CnUG;W&CZmM};wge;~AvD@ty7=(NPb1V9V@avhH!eV{#I%z!3> zh#FHL<*hILyO}%)FR{!S*WT#|z(!)gKQ9Gm5m9T^%xs?wOq=6*^@6YZ>s}Xnj$pz- zs+0C><4M3YR%0=MzCP__;^@-A?Unv}m1-UPAde{V@8DE-Sm+rj_#y?RLn{0Ve0H3uBWekaE2gu9dIfBHkR;Ep{;(0=0Jq4Z~XNgaE&N%l2F@nGux z*F4J1ue(O>2GmtQ=E?Tc{nSSK`j*L68{&}~G=1G~0m<`nT}`_vmwYzzqq2W1 z{3rj3qGytIB{i@^2k{;ZT9aFmeS*A-{{G56-m2p@XhFGhW5iBwpN-L~7>Kq}W+U)) zVM4^sN%4e48r)fpRC@fF)ik+R?i(#{&zGM1G+El0+&YTVvS8ypUwykFv9UabJ*Q_< zqDP#ozf`kN(4JBoA1|z3zTP%hYPiBK$uGq;*7i8aZ#PQ%vEaP}#Kj91)0i7cx+giW zc@nP&!YZ$vAd^bgzL=SsJfC?<9Y(nL-Hh`)%>^+CgN2#_eN4MHB-o5us2y! z=t)eo{sp@MmCUC~p=w?i_hK!OmK$L zE6#6~9wr@fe4@@YC%hT(hm}bbRAgUN%13aHSjA($e?6&4#_LWn*a>=&u7||o6d%@I zLYqn55(G9gmyrCpI3@LIbaGbEwy-c{ler51EDK-|oa?!ECYXf{GZ6NNbVTQW5@6QT zZM;0H+AQy;ig4KqJ9sJ)HxTEupna&hsZrW&Jf(v7KFw2hzW$@3y>2^w;*?y{aRam?9Lg zpv}#s()cV_rP7Py2~!Ij4w$XKgaw_hL^sIHd(Sir5&u=o-AmSITAqvfox9q*M^$34uie`?zALv`T%YPOq{aVoD>4DF?L@F<_L%DJI*cZUM{+}E7 zu99w3XpPNW+ls^|Y6D7%X-1AJ|4>NmN3H68O^=g*8IREheuMBUnG`3NjShP29$Ic? zHrj~f2LrSL)`93%(Uh|B8aeSFB~Hqa4xL!2c<&rqx$4)px> zk=Wilbxz8p$uHs)(f#4i!ez#flShG~S^flMY!!BAxVP;Xfgq`4Pbb@B z@0YnYm_c1)BKZEFQ+YUgw2GF|u==XH&*7hLnK2VnpXaHVE7>#0<#vZZao;@mX+L@H zZ(Lbg1n=0UF=(u-SIlJx@l2L@bywE?V0Vb;j4E$0a&FoepBonK zr{je?j!!K*GIC6xq)nVXj-W1u1}V@NIgSDgeD%K;S(Fr6aF~=A3}^~_<$^L0^KY}C z);xX??DhOb<_#75Hl1ho?FJ7mRzA-1{=USnpI8J|3gc#$weB|898%o$*uQ^h z9P9PBd_kHp#eqMK@R{kT!l9kzwy(c@A+0RzHCj1u|MuKN$9d!D7PIULk{o86o0i3I z5)v+lWgF5Z$#6AM`<_m*CidP5V#TK4jx~+NbB=Kw$|8I=2 z&no$Uy>uSXi-#{YfP_~TM}`tO-WY7h0q>*r3Ps zmU+GD=z_X;Wqii8DgSM%^U-f^2Wa0}+E`iXXF@Xe9J=v|7{sj1epii3oh2!3NT1A| z^?B_GDC@cy?FSG!x!OT@+^OVF1^IV-^*5Ax-n+Os-+w<4H@Oc@U2J=8MtV`ud|y!f zPKe+opc6WwVbMnITV-iAvhHsfDT(TqZY zNsPg#I)(Dgb35@mo+jl6d9k0Z4jOr;k+$t4Z5 zaUo54V%8PE^~CSTd)b*H6SN#0hkXe__P*%wpF__T4mHz>t#Tu|?cJN$vRQiYR-+Ni(wY{h^^ zw)U<`1->yYJv=V@ycT65>6|1cOfWIj%90v0S7^@lQUnv$0%n+Wpwo@d=etBNg@b!t zk8uoc&pNe&X6J8ofBm-d@sD&(I4*2xnO)agNO1nuPpt|Y$4jO_396&mc|cXl+_rq8 z5l&f~3gEI&f1T~jkufYcwqGjR1+=gp_NL2{u#fsh&wkqmWrh9gKDM8GC$i&ot9NuY z&|3d>!Zc#lKcY$1@VMZ$UU)^O=^D%K1FaB8(g+b%}dj~VA2(I=k=(k(c9qS744TDm#EY2-58urvKw zK0hBfeHe~}i!%IpfK{~kai6$x0pPOZufO#cNi*_Q(fUy$0BWV66UbWs*w-}JiVY66 zTSv9mE7she@mLn=Xt*l!nm7Rx@_ou{hUa{ALX4=sOPEYytGK7GrjN^OkqWHy|meGptFXRanSRZQPu4rEoJz)N!9HDPvj= zC!gU*admXk^wv!o=hYQC*j^RhjABnS zoufHARE(8;{ToL|Lc+gp3|CbrrXE=~Idi+j7) zB5wQ6-8SnjzS7*~Fv!irwPIc=>kmel>uk4dkdw(4aellRoTo6yPsjgIt&2^~enlbf zd1F-z8}Jv4ge@(J?EXwMxLOTodRU!+Q}+~;df113Mr-bQVQZ_W2^kgTa_^p<<5*{u zU{2Y1V9x9c{kXS3Pq#kC$hca)-$_v^Ivdx2-oHn$^j~wu?n8U|1rsJLs}nC7Z6+~f z{Bh6W#hvF34E(35Z?lGEE`Y}A&K{-8=^6Zumt>3QJL0lv#W`jhxo#=DROr`Sh4CbAj1wc5(g7`%3*f4WZIlHTuhx^h)~)o%r(U}8$46Uu z&+iX;f?-^bfcQKi%Khh?i))LVrl8d|W>3A1;Ci!)s%nslPh8Y{q3H%g-86eN^`e5{ z6VZj;)aQ9sfR*~)RO%RiaEcdSW={d!d@%q$p0bV%O_#VSZx;*i?Ec15=s&_Owc+z= zr#Vt{$*jXI*+f6kPGv4ah3X$=(8CW)d$^OmRW=-8gX$O3{2{s>^;-=H{2ao*e12G7 z3ZX4H$s!N@UK(|O$`^cSUwoVKU16oDAYh!68%XL?SFQ@F-}NmHxOX9wF6gNr*I5`i zaF(18(`2J~x8Ipx2Rf4Lv?L1%Bt|3dKk}7hk6RO;J<)AxWuJ_9k@Ky3wBOfQRT5Vn zT%cz$eKXNuLwYGY{eG>V?~8qj-v!Nbb|HX^=ZC0jXL+j1FBwwPZe^dMQVGfNjEnuF zmoDj{#i?rQBJbsa{W?j7c-NONfho=Dj7{`4P3ka81Dt&4T?ms;LJ!+1LJU zHvpn^E|U&^!KIh%sSZ9du_dY+JGVTyzL<{C8!x~iaVxC%DH;BkgKC|gVe^6_^pZqE z8&E!?^DX;+@mm}2?d>NC_|+}+DyLoN*`|n*`Y$N>Kh(WtRFv)CHj0XY0-}g?my*&j zbV-AvbV*4L%}`QON{5tmH$x1Aw1PCu&@n?OT?0~s^n1bm+|To0@4G+jz1CiPtv#RS z!^{=udH(A7#gW4r?=AApIZqX|@vLFFV|;JRj-aMUM8rYY5oTce`rux7++ZUaU_ilo7!}%HCU|eImf29dfb9p z*3XahpM`{oCElmP#R>TDr<;Fx*Z;I6Q&h)qzd>JQqoavw1^1Wn&QClLKMr)BIB`%i z-4=zFCipHY%De%SmIS}p1mynf1_n*2Xqu@rGi~Dcoskj}T=0y$f%BaD-=p^*lM$0VYyP{ZSTiv zuR=%9{fUa;7)CjEq-xcp`$`Jtc~7*od)~~aGt^2B(M7BlWZnCTJ9QE05m5&dwYLq? z_h(7;W?n#%Qn$hAUQ_|4nH?i9E@%Nd-hq0S{`|gxpNy|}g>MK4B6}!Q4OEN1-S zF6-nWF1>mMx%-pbMwjt_Gt3t(b3-2G{RoBs_?0QHsjCn0_1s=U3TgKnNcy~r_V}2k z*e6G)rB3EKJxM4mZT6m1qeHpkMXi>-z_M|}>9C;JLFcbWXmy1$?=+M-Z!OF5YC-o_ zkVevcjpX7xg|hhrz64Y`47HV*@%nh4y?^=du@{TLNcFwLh#NPv;)I6aDbeuHXRYlv zlPoZ`r%OuOmZ+CkUIPC0%zN??WyuPs^_e=Ay@=>|y*K+y7f79cpoI&_q6@RmR%cpO zdCQdLAj*3gFOe9hkeIIw?xAw;--gd%X$1=9DhpbTZyK4K0~J*qqAcq_CXM;b<1-uc z63&9>HZw6HF#oL(pRcMdC>J1gSk)>cf zMU^ygICIyM^+XrjCDyQT=0a^EMKMvC_@ONeD-?cb3gRCwp^N5HfE{>ym<7!HH8{<%;ahsF~lFzm0~_SaFX#xqn$ zAKk5kQ*y;4FTzpgSQ3}ZkmUx4heO#ve1HFGevyE&_<|yJ!j7^vokZeR5(nYST>idm&vTKe8ceG^!k`kBcW=E2*-9J|P>*$@sw18(b zUmc|MwCkSeBgGlKNOy>F%p{{-tB5=t6P?Pl)pe6v^ze2Ofy8c<{`2Qp6%MMno*zL2 zjS^i~e)~Tx%J{c@MmP~l9#sYMXE&f?=NMqJW%Q?|ljG88SGcdq+Y48{YC|{WxCY0a zer~{7C(HZs=}s3&)UlY)`u(QwL7}aevx?lAU$2dXqs$eA7juy(6P;o7u8YWi`|DcJ zcdIR+sT?QZZMqiFNvgV~T1DLiQDUG-Mkz^n{q--koYfXzN6hZFqdww^KpHh#_jbnB zDb|yHq0J0lM$d07rblsJX0@CKB-EB3NnWJETUgSJ1#ipTW4f_U^ol6VEx2vxz{jgI^TBQlg6C`pepA-jilb zSNpYis#&GUTWKac2t`E+A+n(JyjF6!d(bKB|rmYQkw1oK!z72O}9HCOIBr;@2z_srzR~RTv+17@; zZ~k09svT1J3XDv_!MW@61^`S_M%N)B9hwc^bqzl_k%mI5&6QQS9lgWr6Pxk?;O8r3uFqaVpE4m;;5Y%t$T49}xu{oRPHJXHk&GeqeY_i&~M- zEOFj4!m4z6(Q;68{Mq^2N9(*so1Y?Sw-cNEbQw%8Wc>QYtjcsEni{6>+!{Dd$xNrS z++6779fSQCeZSNjF8RJc!@n#kq%3IUSImJ5C4XGBMMFo;YB7h`S=-(I3!nF!O$ijE zcieLxo_olwCCptdwguOfU-)>Sgl*|xHhpI#sW_NOTJRLich9A+SYk9v1}FXjx}POtxIVua}=j{rV8$F5yYR6nnQlS%)d zm)HB!WnNj&ZOmb?v0(v}rjgW7vOgLn-E>u0y-YJ}L6}#PJgW@QeNHM1`2D18@$mK6Iwnqh_zZ zGF_)HuD%5XUsJJMeT{u_4}b|;_}2j8NFxE|(wHj} zd}O6#?DF~SEH@%wv{Ws{ZE#RXrFuPo-g8`QWO7b`K0v!P!~SC~@!;(eC3Zr`h9l#W ziZVN3zD!S?%dp65kJtQh^g$8O9-8I+=ZDK*>AR04%!KSFq4imDn}QBA%UHg)6Rn2c z4_(X#X~Rf`^K`LA*r-JpeHfsx*2J%akl4@{gqZKPQu)$m9yjSLY<>FN0t8f*<#0+! z6i?2~6`~jdR9`=kylk#p_;Yg<-jxC_sb7b*^aW2Uq|iCh65q#?`kt=>&0hgn;oo_F z5}BR@Jn)K*)jYUP3kIK9jwNC02$(i|ay2*~$5-xF#}v^D9N-Z)7+k(kpqXY47&HCE zeYu|n9~st;qV{~Z`AV+Eh@CBwu<#ibjN!52l+x#s?3u5P zz<@%=E5aRs0Y-XdS1p6h9T;f;XtY-rX`|rXwisP|W2p$BXz^}>7~FY}{Xuw8dB*C_+28J( zA>5Z>+RG{NBgMLPvCUKa&DJ!B8sNB%tydYZHe?)O;do7C+G&QBg6OlHZkk#&Hq+#? zv7_OD|HUyv!>`65Equ1J8uM1fbXb}At=-UGCosUr4+@^L0p=oLXYII;>%B6u7X)i| zSuO~Xrn>GaY$Z~16gQ)~qep9N>^7o}9GtAYqtc{G{b1m8G`H{hCY<1WGt~-|aa7%c zRPdeIUtvPD4!>&2`#pnLO>fP-Fl;k1-!40HMOiAuxTDfWE72oJ6W`YJXz>XNK`ftU z&7jQJpL1t@6d` zwAJ!n1UH--MQh?y(Sjsd@hRfl430qk&^k(0|K^PrbEsQ?CTR;ogH|)S#Bv?@GCT-; z4lXXeFaoD%6sop=^MB-Wspld+-oq9Y{UIi9-zp)fLy42Qm9*#+h^rKI2Qx6L!Mda* z!zv$!*o2Gs;ReqO{HTR<6X=@F+Mqsx5rTt#gK zzG@GEGQ%q&zBzbgqAqoKusU&D#dVlR5ynOs7ZZ5k_e8ZWs+4wE4KOWwvC=MJ6o`+Nj+1Ro(HmbYQv{%7(WS zmmhUJPS|iKoK>NN8Kb)SN7l;vclLsr&v}iPaSL>A$9dFw;@xY}^^-Hmz0GN>2o+k# z%=vZC&Dhz6wSAEYHr3Ib@)e(qHYii0OAX=a#OfrDmgu&QT+lNnQUXQ(fQdhH?A82K z!yCE;p0MzN-C(?PwJ}N~X9k?Q= zAtg|yCRNVH!3-pikIYhax4l8jPnh2Mz-!=QNDlB_6b;NsIIuwiL5`@vD0L`_H{_Yv zQH~PxRl_jro3>NM;4ZInKZlLg%F($>{@*R$pp#=8N#eZm?~uliYgYLkUbNn6eo1^W zTsSzpri6flaPrD);ExREr>*DW+tKpR2#hBL)`6bT9~IfQ_soTSt9z_i=6;YHhzMbM zHi9^8w|4*f#&XOEl{yjKh%i{k-=By+fkMCKjvfUE=Ajk1DLz>rrrZ&xsIo>2Z+*W7 z9#7&^<%#@CZVrZF zBbF*-M3sOz!ZYQZVPGG1pN-;{I_jj^Bmyoq;fpUKsdLM8N(=n-vvkoHa6cNqM*i3n{@kn1PE_w7l0)_-#`vS>p`LpOv7 zM4A~)iMT1O;KqC4W`9uUz8h1y7CV7ud!BIwZA%W7M6J|eHg}4WwRYMxnp7X@^Atza zS<@Ah&Ss)s!qnCUqY6wde84J_v?hL=oTu*YFxwhYb{|3s5Gj=1(CJf`+$h}pb^M>hhwP_T@ z)7^o$AW>YurI17fMP39%F)R{XowC^!wVZzr5B>m=Yd7VV90xQwN8GfjQt9++&c zmE2K6L`g~D?dI9$&>^pqv^Bz_D!ZY5`}Z+R)iAaoFplrxT#EWr=t$Q3xjSJ`QQn(} z&%owwK3hDe?ptC>Cyw8}94YwMj}ZHtv5w#)3RRmXkCnI&4H;~LOtjREP?DhPH@BX# zib2%7(p|QyvHBhR@T?d}-~IjLwE>SZtxwgiiW`%N*z;)Nm<;oq$SLeZ22ydgWnGEp zJaSAB1qoyYpbClWG!HAtP+yif<5IQkK^^1f0Jtzm*(WiAkRXoVsWdt?LV?ERlNeD& zqjTwCdQK_mr@>C%cr5T-e-x|;;aKat#-x11Y;;8+#STv3FI4g_&L1Bd<)Ee~K>Yfo z3q#NhuXUK(+D!qFn;zQGK5giD)9q1|r?sNn#Cj1rv0h8e^#us?y3DO^*6!CqE*{uT z_uHL3^}qWLZp)^W?Fu`2m{&Rt(Vt+I%H^0E*tc&2e?&fZN@{+r^=?on$aan2fJ`8H ztVBV9HK8;5MrD|{I%$xbF5riwMXEM>FS=kp;93f^dN7{@z=DN88aLIw@dZ~ug#2`> zQhRNJ&Iny2p}x8?4k?%I%Q$8LaDdh8bI*#pF+o3;Z0M@C`A6K66twn^=V6$UqTDTN zVkQySBAagG)etwDvKvu%gb?AC$~$>w6r$sj)WgyBmDXngUj7sm(?@oo3GR)JX?#~T z5a`L<`WWqxo!alFH#qkB!a|urXZmP0U^!0RpDNZXus;q{ToUIR+aJ1nq@S&SiJKF-P#JBnv6QXauz`=rBGp z#dWDn)!eMjMINx{<68N+k0*JLu6VO&3XYHaS|^tvZGuW`F=UPg7_~IA;%>8FB4;Z`Vd%2ZeOiKSsECw^UKU$*Bx$6MUy>X9*dbhc|a$B}Z)pr;E z!*gdcO_1kR%xxUHH`VtD{s}(*KZ;YXZub9r@bmxlqF0eoye6QHNbAv@KVB)q1^9dB zPItH+vNYOPP6bqwJ{6TH-FI=toPX(Gew+;*ei~ALYZT+4Sh7>eBf2OBLLoR6<-r|2AO|9ARK5EIzYG+S3yWToNQ4pqtgQIy(i>$;z zMToY;PzfEM$4i<8;|o#SY0VSGo|=>d`{;GT`|l0 z#w#~AKD@Ne_?z|kYyp z&J%~pGf|>|=xROWx*+)}y1LTF{!5(C4rdUs{KyHgLlWhsA#==SnbJ#qmo$WnN;e2} z;&imiON}|Bs^1dtV$LeZ?>$g~_u!fIB$@aZ$zfAt*%<*SMDfM5GJtq(VaxtB<<&DL znivI~f7k8cir!C3@-s%)5n7R0eY(vaw70i~OscetHxbw4L=M<|*~>8=HN{RNTotG{ zXEQ|?;DkPs{nYIx$P zy{{@Y4F1VNHVwdOTCR~OscP5A^-f1ntt_kh7io#C^W$dAD3^0$tvwX$Ag^igBqX-3 zmMU5?MsC2|_dPJ=OH75$mceB}3vF@YnKk1Css7+~`5_e`!-J$Rz!TX!7mv{0!qomC z)OysvcXGV~6Jl||q(a3)CNY^}dWI{fpVo(3+lq}V{b0%~T!ej@d8*ys>4eN%UfteD zlcf9lT&ZGRtZFxsKVZ5ryw19bhLUB9>f+^6p(*OEs2a17ZKd$kpVi z0q@gbczAfYB^`(_&`I?Ns#p6GAl_tJUS9q-t25TckkBqY4YN#m>AdW=242NPA`g{nqEMrqR4?)O!m(*Pz9q&EK84y`!q4a3;1T)uf1RH5Br9XAp$0%5 z;cd=Dm6ap;y?VdJpye`D?Y0@RYAtdLKZpfdy(*5>2V|Ydb$le7LcE7<0x8 z(cQQk0}fFw@^-2=5S)}-LbPeu&7>sA-WC0PpkpwmW{f6+M}*8csh2c+94Q|Es%Kmd zRJo%vgv;yg!!Y?Po3oG*#dM{}u~%X{bjuP1)nj6aY13j|e0gerTNN}yVxy4kwG4Ho z*gBHUd!R(;#njCCGAJ{LnMUuh$NKokZV0c<7abPDqrS=Lp5)`Bn0WIeWMULrHoj*o z1fBx=kEf@9Y*bq3Hua)O_=mqXZs)4fPB`g#ln$7%Kl*E|#q!eBZ5D=JxCQvm1{7Y5 z)J2oUc{tG0Z$n0?d@+(n+V4M-#n@1O29v@%mtLmj17bP!=z3M8!zRap>Wc2o#6$F% z=IaFV2ZR9Zo!VA99PLwHT4tVN`gjcr|K_fsSSgV=r5)dZp56itXe}m*`t$2fI6P3w z9e$cX3XT^9RInz}Fav-{@=UOoh4A=4n67U_UWa*oAe(q1q!6m#lQI zouUYrgKo*Yz(C48wGqHvSwFl2>So^xTYPHqvwB^N&(;@dWaAuYu)=YF3L2J}{2*S| zA0`h-ewfeczr^J8*Z(f9~#cb@xvRe9*~k(~hmf^v)Z-NJOMj3HI=^ww+sT z(6pDGE#66ZK+Nh^F%+FkNRL^2d|$0U&C0$B88jmsOUlW>SuT^ohq<>-3Hfw!HOqM+6tOAI`_97XkIdF4<$C8Bk^hTz6@ zmH0Egxuot*U$|CSejz;v!9|skC;emJxmL}|Cp=K}3@_5)3Q8x)Znch}iRpoVjV+(0 zc#@1}c1XzmHrr(4GQgb4SpnwkZsv|DJ6|2{R21B~Hx1c5RD=#aO?ddvD_dQ;LJPSh zobGCBxrgEr6M`z=ifSVo^xM0nUNC4K(wBgV(8tvJTXDH!_58QnHLWYTzv?lGNF)Q2 zT0qy$*)Ywui;AwD`W^N_N2q`KvR*bc*?Z}Cf1gDG-t#)XmIhv?O}39UbB1ho9V1gz zaq%DOBgwXSpwGpVvNn^BlKK0`@ER6K@$={4@?b}miMz`q(5Jm5^Biz+S?xzo9oQfw zsl=MnVE<=|dX{X^Zqa-tKpi^E^`@prG~6R= z5BpER^rurDELQ!mNAE8W^@;$-RtIptk8h0zieG#kbg4R?j!}QMb#M$jsjgvR@`*g? z>%F_v`X>;ADBEa3GfyVG4>1HvG_|D^CWHZfrohMe z?8}&>o@nZifx~F{Hw5g59g-vI^1|EMj&iN^U0J=P4h?a3maki2B-K*lsN)oaPxPLW z#>>)?|JjV|(;wO_9b{i~gs_UzOeggZiFvat3^s(H%@@emT>j+f#bPf0##tKU1?vMX zFQMN`c$pI2=}R0V6@6X%XRQ48#X=gm^qRToWR=AVZ*kT*)y+fTDi44kSI~^yCR2Ve zyAscg%4*vlW_#Xy)YE=fp+5tu-P-}3?5!I48`M+Ik=$I*^lnv+{e^#+{^*bBDf&fx z0jhiA4iPKy3Cx_*F~?oAF~ob;Z}3tl#ol{I~EcUYHUDeq}Q^8(K6PpD#mxWyPr(MS&!v9gS{fof0uJC|J@+3|* zzc#$rr>39vwsM8QuWg7fLOxGp?T&CP7}eo?=JWQ)i;`z7bUQzmd|;jVGHFk)ERN9? z^D7;pey1jSh%RW>L9ec%*R5ueGOs96420hZr{~y&%P$RjscA}#vDcs1>Cb(ge3k|Y zW)kXW?(}l#?RUVV$Pi&TBAa$!)hc6lzxR8IM~}yGLTnEA3y5~9l#||T!NeWTs~Kk0 zX4l?TvRJc}a_+K4)N55q&9*wDNAEg-*~L#CV=IVBJ_X-x`r%gPT2q62URDII){@jo zahD^#u|6pC1K14dG<#{l{meEmzd~i2b2EC6o(d_cLq0SOEmMBn+kl4k({fRpf$|)V ze>k!&WnZG`9FdM+at-TMdFdBZc%cNBu=ZiX$xY12cwCZ{R8a3ns$@JdU+l5KnqdtW zok@#nt?J1SZ}e;JBNMSH?blthK_enr0%+)iF#JdOu{5ww-MBe;wFJ>bHN2%5lE=wH z{*S#=zp{5ktQvl;Eq)$=qUQSKNV;+O(q3zEK-R?)VZ@lj$l!C|G zx6nL&H7?2^sK(7#1*c?3BP^_Y^pd;Bz1K&@05kCX3%+v)`uC2>{b) zZvJ?;Ie+)_WEngKVja;NAV?l@ZPHS10S2CH&1ogHkcAv*xo+h2o33HxSPb*np)zJV zk)$^D-F6PaYQ82;lDE6%f^;2+$lGCR97_m4uh*h}z1k`3sHq*8Ls6c$K0}(5rj|Go zs?ZTtm+9Df7Q0HLcI3Cf-t)UGf65*2_m_k%^*%aX>M`}P>i;uhM5;41x%;h2^33_7 zltsivzjZ9NO~pn!Fx&q?-au6d))^?DHbZ^ofXrS+C`4gjyI%U7@9wee9=_WtVQ(_N z@K1#NJ@n5oW@1z&($!u{%EJyONw%yc9il((ovMWO=sZLw5|GeOza{^|sYPE>3^)^w zRbxW%{BWq>4Bhylsfed0Jd$cFj)VBs%JR3Z(u$x5v%L8~?XrA+(#236BoNe0i0uei zC8enUK}co8c^EXBwOIXqi;NvW^<#H(K#~^C0xHZa4J&@z_0;xl>~GjJ?x;JW%|O6L z-$gB{U&Ym^D^hSNs{6n^_(@mN;{qnh3-tm*N35s*wn_R<68ch;^Qh$&*HVF&4w6c8l^=rpf z-6C4Z5#I5_=gW2h zRh5U{OC654K4|CLDnWO7q@Qu#SUzQrw$=eeCQz_}_}Ux3PQ~LHNzswrSJGCG-YOm! zp$r1t?XxDNGYgK`ZX@T&(37~=LE?~SFvN6m1VA(zRZ$t2#MXPCm~6w0N6qh%|0@$6 zZcv-~PsEMq5MzZbRkGiSfZyP-8i*fag!MzMkDq$BaJs&HpXO9eS5z=+pq<~*V(!oC zdr_rIoy8b)Rz5H%xFWh~@oD{C_o%`|_h!`}63R5otlb7p9TRT(kdTi*H^2rn$8sl< zIxtvyL1XzeEKKri_A(9eC+@%JWaL^aadIOs)?<68v5z-T^a}Pe(CkZ58k88HPi~L? z4n4j56zt-0eSEvwrhphp=2kPsXy6#lu9e+a=IzehJU3len^uF+ByT3x{_)?7b)%&u zpN>Tw$PJg^uENpe3yHfd-_!Y(Wffe>w`-|gjhtSVIroKafQPfvHmhY@cwJ(^ojx*Xd1zqw1)mF&aWoP|a;o{gP2%l^}#$tddrecGj`!$fRpwfl) z&JF{i!W8vYsE{!vf(XC+Fp0%q_o9~J*_=b=$9Fp!-%W#VF+X!l5-gkind|F~vx6v7d1>nc@B+1}FkIDsw9 z4Rhk>{g`seqG>6wfU^VNV{GF7<1%ivnbp^ukD9zsq=v(Pzu%{su*ZW9s8QTSEqu>b zgG2(h<9p?`LnsG+lhESl=v@0`1>#SljfwTiJ6}2Js(L#ay0HD7f!P0r zS(&vP^Y~q^=LzlxLD-Xv=25}QvwYrHWQr!zGW zgYUTEj|`#~@$tqyax)B#Yz#~k%3Rw`eZ>Ks_pm2F&Um^=Oa+^k!t4`$pyY=@9Qk$g z3BVhz52iQs=_W0?wzqXi8*-sg|Ac%poy<8v`l%@nA}O{Kn*atW?RMK@-k#u5K24Xg zlE2>A_gZq^Lbf-|Pm{$b?UbKip2-@tX$sAVZ9RMw3a~BD>OPM%UKov(^LBzMtpV1e0_lhRY+=ooNr5 zBs`n~G|bjH=CuUg9mWN|oZp#wHA_=vLj4yj$Aq(@XyX|2jf9b#$pj=V!}6wXx{y3P z_J^roIP>~ICQH)nRKA6It)@P&lpoY?{n-ZGrqucxJqcYD#}2nm67%M*c=MaDsa-Hw zx#ae3>A!0{j+Q_A0>m!bWWfc5t zGb4>=H~>%W83BvMSQ-!<0t*=z7bMQ&bbvaIAzMleGcFOx$?1kf3RvI=s}+-nD+Wn2 z004{)uTZdRUqiQb)`y)+FcyxfGc^cF$il<}Fu{^#gGEf7#HoaLZAwr#bsP|DxO6`Z zx7he;c-p^oa*ofZS|tIFn6+5_nULi4emXRu*{a=sUwi4hk{&Pk4ACk5SVY0Z2RK;& zS^&(o1t87=XgfFzSS;TZ_VApp_PNhJ^4Bp}diX6s792+N#w!=QLLR7@j4Xr8>YEo zk3_YhNgVa9$e%i5mW_ed5q;AzpAQ}_<$aA3l}gq<9O%ehHIpr{R&VS1>bCb|o<)vN=@RRka6>rooE?hfV>Ss)I0pS|Wee-D|= zl{ot6rCbi?c;y*AEg?3cnahBV1<9j6&`PP|tHR35k3L{R*~P3zFSn00rVUhk~g z`O86h0wZwem&N-^#41_82{HPzljQF=@{C>3hnZ;Ve3fGF3jUlwYtffk5xGm^WxuQ^ zW*t!yD5!KLON`g*ETMMoV||XaEgYPXOE0^GL5KAQ?|_qfms`JyR`m|7e(qsqzq$9@ zF;rP8PJ`hhAefCkb&{j^0LE!soRF}bR4qHRH^eTWIZIRlvSS?`PGiA8`Z~FO*0T1I zuI+Td{7TTF)cQ_Kt3KJET(FtujH~5tUoXVyOR% zCmCVFNY)La@y_~PuuHAq)im+~%p-8+L`oS9+yH{vhhBLRTwA)WGf5*-?p}Gv zE!i3Bm=Et8W;N>|6-MK~*2_YbqJ=mDU)ieNS$CGVI8O>_cI|eeBph>?we_AEggj#& z7w?Vv_aO+eNkHd$eC?Xdx7NAJ;0>21i*2J0%frIh3~mk*PjQB<%KJS{9)t-WKR|=e zZs6I^19czIL;wBzwNEOMo$<;jSV~6*>|kHRp4aO2CUwAooX9br`*_7!!D3gx?QFN3 zxd!=-O4P)_-R<>l&}7NKSy!B^Q*-c01D0UaWSfY|r+Fcwvn|r5n^aElNmPl{Xh@z! zP&;(FoO`8rx|?K!pvLCM&8J$5&i|&aQlrFyk{(}0(SpV&Ke~?h4@E^B-^YD89I~I* zCQivNftj`QljwYQCtW#*pb4D5R8JUL!nEc5l>v|f+u)e0~LKs4)WH=xC_ z|9ew|(NLllDX<4`biVW%FHbW(N*=2;9?ZRO6aH^DdH*__Oo!-$CiW*y%adZMlR5oc zykkF{dDV^ygfIiiYT?H=4eAHXCm^+pst=80+>;rvpUwX@%UR38iD1i@{5Ru1!25lb z3@9jYR4;PB>NRlddm;jUnOXkV96obgZwmk~&j>C)ReV$8AN5V*@X*ClB?3wRqj%H( zNPAxo&06m|qqp=_grh&yUH^N3u5~`{2!OrpZwO>eaz;GueKFe0Mf*E$-dI>J!fz|Q z>b&O6j%lGeXCjU5x}CtWELIGkQ`P5xyT)iAIp*hyDl4S_j!u0eM~NH7Oa{frWYO_j{B*5>!HLH298C=j$W80rkrk> zZ=*A#x1pG1Z0cD$WO#V>waMn1YfTH>bC9#Xr)lNfhUj<=DDHM#-#J;dgC=%76nnMZ zvd&T>n6ucaV*>VvWF22bLj8btFDc};H}lXKs8MYATUrKbG73tGK!_+HP7xkV3ogCo zDqL}>Mv&L`?=Z`kq8S$nZKtynS)wSuhQE#ecmS45yMkUDR5pGhf9fT!q+ zGJPDHB%4NHSy?7-uQ!is{m#sNF5xh0<{|T1Cx%z^ zLC5_j@E_UxUXTTz?x{__(-&lhKrbM;zjUfOTB82WngW#@AdCTX*ZFE%lBZ944xYc! zZ<};lDFuk^pZj1eMgMwxEe{G|3cz6|VK;4iim5GDt?K9f+X!egRx`b}s%sp8tZ?%~ z32aKm!mzq1QOuNJ5m*tObcY9+tBUg2voPo_=Fc8x<8Oi95ysRB5yYhb>aof4zU&|r zt1|h>={EmnWxs1zbYE{I3;i6+hp^%1KHtycU1>dC>SBk&>P_CMcdaqU{oMj=?#?xO z9>f1(%$A{>8CZAG6ACWdFayys3JHnB`nFyYQj=zt!(8Hf*2^g)4aoFAyDqLKEp4)8 zV_Xd@UyU3_!wZwuSdC?sd^xFYryyiPyax8@vExGrf#g$KFN=B{P1)=zDdi6O3PT?@ zR5Y4UXS}<6?h@vTtjmcf9A*hnl{g{aU7&QFqI-4g{j4_Vq1o1IoZrR3`QOEf=d_vp zXk$r_!tItL2lOQpDQ-pU6MQ@{d}{Nqf*z~XO*Oi06$}47^#^Ga@xVOPYI8D+Q;*>^ zB|+%)Wf6|`j(b-$1+!H;1myr-{6K@66kQ^E&dchA8p+yJqOE|Bt-AYJ`Kq;CW@Y)$ zg>GZ&yjFL;hlwtkh}$>kM~~B}rJL^JSX{U3+=&IMzS(FCsP!sB4J|+u6f&8#6@P}K zp?qJ)$q7Bt=SOHeY8Em|{!*U8QEEz-x{5Dj*@~ha+$l~gcUVZDBtM zLIit(f;WEUhQ6S{MiyY03v_aSq-npk`-2@akHPGD<-?2&Eo7`%^CL5E%1GK}#-y?= zA z)mdT6Wt~R?Sxw;eCpRfd{q|6H*lJf*&tmB&KXs%5toYDlLa+^>Z*AZ?@!2p;DoXKx zSiSSZi=ahKvK9|Wlc$DiRb6!*qm;&vDYfkHo;?{|*WqqEe!mHJl{o1!w(B0p!(UA$ zgoax=2eW4SP8jF(yXf#wH?w#eIhGeWjO(^%j zQF}EBNwg84UVxFd7}`ic`J3&0atk_+TzlyEXo)E{HidJF-e8KyS12SlFek2nF_ZP zU*}!zc!)XYr;0om8~OeMLa1Eudv+8lKDpd3uYbN4{89PV+lpU?lmuk%)(s;i9O?4~ zGcf7%2I58!d3L{y7jJ{yDjdd?drLQLPftNREts9MBb!s78#uAgi?4thS{nsp(I$kv z=y!0KwB23QI+@*Fqg#RU!?s_srNa~5JaX7RYb!m{=IP77i^~xvr(Y{@Z<^JC)8M(B z2sR-3To#Vjb3fnp*{&Wp^Zs0iSg0ML03~4I2M+8=*W2HpvrP zIdakdJc-HH(!GImzztMd(*-qzM|2wLu+mHW+ce!%Q%LW$cKZY; z-V%;L3cKC7D=K6_<+{C&tP84$2wB^=9Ule^2xp$?J%PAX{y*!_!4ah`*tga<)Ap)nIF*C!@w+LUS}yCMSq_ zz-7!nwh9NX0vXe&rhuTY$fJZS}DrUc| zwP4zXw!ez99#xY9;un(+UL8Va!OPy%Q@2+fRI4ji9PIZU+oOvjxPjv!#od$ z4}QiC9~~&WsVgd&5JXLsDy-I?dhoP4Zl$-82|J8RFup0ekCT1*8aUqMs<#w%gjnVj zE=M}8VG9anpUp_KoY#&({JUQb9I7*{TCB%kS!!w7-Hz9Ouuh~;?NWgNoxAza7A^XF z3WZYNjIQNwkh_!kWePg^6L-FG=%o)yZ1qu$??7syIIx=Z>6tUu2A0KIpHKAiB8Gof zh5}D4AEvhHn=bMcjfR7QQFyF}eh4cAv|yfOL0qcmMfMf4bCmA_%# zlcUWq;f)lpD6_UxFcNK`;I$>oj~Qj-v-%6?s&R^2r`oJ?AmAqVi^?1lECj1@qh10v z>!SJv!dRei17)>kMo*?&9f5=$dF2J>){?2&*VXx9t;fsmfYk#^&~+{FY_ws0+Z+;) z*3q_z00!nhX>24Lc`)4h&0AJurrG?a;<@;t$1qaPH>%Y>;$-K?vEjtz(7aC3{5P>^ zt>!pg?*ZaW9$DZ8kXj}tw2yGeZNf#!jQ^u^0fBunzB4%s)<5-dGS-RI?pQx_h zR0$KC)v_rNq>YNDdv7;X{I17Y7Z3gDes4mPd*T%CYf_$*(H*|PrNrS)rKw6x3I}u#^Uqf^ zUv!=*gk}DEB*pN=o$-}w5;7+j_kS_>mT^%=eYEHh2EqV>ib|JA3kV8BgS4Qev`9%e zNP{4VAPg-zARyf_Gy>AyF@$st-F^1>zW3g9&i!;hojc$7G0!|Z|FzdzJ0L}XGe=#N z%I>f4kvr5fDVwtw?f1&D-?Ly+%e|Hq$i-qN1XN_Wbru&RTh0fh|D|GBx;|1XHI`xO z!WBtKN=@(}@83VW#Q<;o0lPt1h(}A_a4eJqyp-sq z9SG^ZtROhAZs6mt`Fat+-QL|GhtSNG_C_S$qP_=0GRL&Gm$VLfIFN@gG0lWAbkrFK zCnZt>KCkY_!?f*4C3J^6h(qkt6i+?=Y2)Wcu+5FAXZ`uILp4;#KP5ZUI~jf~1+<|lmHD>iiu0%ySFk~O#c63vLl_c`i@sea=C zJbJ_w%7%=8^Lc2BGY={gM=%J7E zCW$yV1vy9!xr7XI=F&2Mii@*l-Ga!1E1@C%v`qfPu2HvX!QgQZ8r~B!Sf}C=$L6eK z0HQ-!5C$@U`4i22?oR+%hxZb_>lX!2Qb2CQ5S-UW9eZKpr9!uo+*M=&kpU910QT_3 zv?t?QyQasXZCkA)}%6SqXqZJ5LXNw4(x7VHGFLwFQ#j0 zAptvs4uH+)czS6ZaebbPVjkrQ&sJeH;(%~BM=`4y^~Cd*d>o(y*C~^MaY_-wAb0+K z{!entF%9@X!`WB>YXU;^s`l~-RWr@!dFIH>!4Z=j&kaZxdSxHO!9P>BF*zWFhA|Jo z!W9K~|2TW|Sxr8#Ivmwr%1R53*cx2f3~HnH)|qi#p^W3TP!H1sBTJ)#v2+j!^GZ8? zm}Om5Y;4!X>Auy=(?4M0c@LN?KxxMcbXmbSRv+F6`v@Y#{P<=4*?Mlxl=H&(7oD$P zOf!gFpY|CqcgNKbbd09#w2_4k zH3SH7lTv4jB}%tzwuR3hYv_BH*Ii!h_suTxA96dR-;whjE%Jg@Huy*agyazT14k;E z^7oC{?)i;~o4#%}0p1QPy;d(oExYij59_O+eJ7<@cQ7+f<8a&Wmmr#$n_Wkj_EpRR zZ#?FR-S$Zz7&n@v|BNbonJ%7o?t^F!#WdfPE1hO>C_}k&NnMNS^Y_CjrHC6OyIIe_4sRzxlrh&_+BG}@W~|!qHXj(h!Oy+OtV9~+6&qE`{KR~N_bA$d zUPHZ4b2r@0wz-n*mcLvaAsZ-Sx9 zPs+>cT-x!ocb=gYh1vT0UZ+c-Q}7@RF%tYZDLF2PqwcA*PxbXIQw?Mv!Z08c6`{FhgjKJsvqORMbTUDA0RNHu%( z5xv&HJ2f?9oIhIxe$Lo`k}tf3JYNIW-D(>oI}k)gU@V285O)&nqhE%op)p|8WL{ug z?!V9f@61Yy3TN%kXE1$Fw$3#)G*E4rW?~#H1VZTsE+n>R$E?cAcY2GVJBwjdJ44d0M@Dye5aEK4BY1@OUd` z!2OdkI$4w3?Pr@8^v_RBXHvA+>AkADE<-6C7Y#~IcbG{yScoAqOy8aEy6(r_)Z7cw zA7z7fteOckQ!YghdX^Y9X$VF{ZalClfIM4gX;#*_8`0?`=DLz^GO-71Fgn-aHWNW~ z3o_~74BNRadIJPhm`u?z7HG_8xrzn2ep~F%vnzR8q-k=l(GGo;+5V~TJ(iE^ujo)4 z;i0F&B82V!5+%mn2W*doJD4EV(#`EHHoP4%#$J#3Tj`k)kxcHJ2=;azT)5~>%#p66 zG-_u+>^2_Fh*`^meAsXojFY!xG0njkVQ8Oii+k4o?Ks|eM*fidFolhUj^|-;Kc>_2 zlwZQkzu8n$g!2wud8mD?frzr49!#55ouTfRq|(vpUysnld2!Ft_s2e=if7m7xvqFD zNL97T1Y1OSTM-*kE&rRP*@+fj~BlR?878GKYzB?_qPGNc~#;O{FGR}{MXI$3^CzKiOFEUyPkT-As7MpF4 z9JsaEt}BvH=if~Xf*J^9{Iyb(^etKjAETq|t^8;-cOk{@G&a^VTSu9S063AJhLsWl zOX`NZ8?ekZXt|!^bH!MMK?dRuLTriD(Y=wt}NYWjIdmUvmsS!eThP}WhatbiYdy|e-L#xS@e7m7c1AYQu-#;rE68FmfN|H?UB%cVZ%+It6`0~G(Tbz z(UKqM=8=N?H3Fxd#*EHX0+5b{mW15JuFR$O*oJT>86{k}%v1I>umAVDYogU-rrZoq z)Oq1h0D0>aZIGRNTXv~B1wBq2!QOQdvFXr}e(AxzpXl*Ou!$Z5Kn|3`0w4e-c%Eda zxDpTh8-o2H5i8SAK>evg8r#)N8dAcJ7_Onx@c|s6 z6obWiWW41=e)|w!;Z?)?LNQFP~2j}CwsVOQiXnz z@%cak5+s~)Ka5jInRB~#m?1K(H`+6_Bcti=Dn09rMT{%wNY}7mN!C}~{!EhpHW4@9 ztM+uBlA!LU^6%0ywa%~t_ViUc=v~pDeI94lrGoYj-$P(8rH>GCN;|h1+G}GU8z0{K zZhwK?|3QC4wD3^lWR=Drw|^E84+b{MS+}rdtP^5y8XYz~&ibq%k$5GBIN9)Y+B?xL zw_gnVwwPi=B8(lBeygwTN+YOtPt{M*UjBQ&;7y<8)$N_PuXmxb^d(Ow$8_&ca)5Ev zx;&W?i8l^w&iaL|?5j9k;btZi<1xr4!-)%Dy$=Buja>e35j2G+PzkNVTDSS|_ z4~<}H(Du4)mUFj-LZ-Qh|1;~d9Xr;`sm|J66BedrY z9y-U<`bdV=9@zTj={y3vF2X+me#5YIF>RABaPOtOxj)^*k!|2w%+6lk($qtgP+_FP4kEct*|Oqs!p%pSc`+?n5C7OlO4S zx_<_ui1@Vtt(Z8ncE>g;DrBE4><&Er;Lq=L&SkE9@{siUIv=IKYf@0F#&KTQ|u=Ix6kH}ql1)FP-$>G41nw@n2CLela7@!K)JMfW2=0@lGZ?spZ}nfiF6Z{_6Mw{;_xs9HT36C*AIVa+ z@YAW%gz;F8-bwI z`EYp1H)UQiSNnV_z*j1~LBBzF%gYXg2n1{Qgj==ljkA?rvI5iR*i~+78Jotz9P{~k zvN%hUZAe%}a$SLPL2aRGX8=#Eg+K~6iSA&K{uprYsVf=ICl6~Rg8kQ*@FV4o-wfP# zM=f{4r>+#DFSeEhm|2~>{@m3YSd;jv@*OCf`hyuLGw*+$?AZx zZ*npa8GVWc9E@XhTpMzUU)m1iZQqT3AkQ*L8B%~uH*FN}8L?mWl1P6k(zR8lZiCm^ z<2^&GRIABJ-z8|WvK0PlnccaD?uqR^&0>Cgpbmf0?bx^Fc%@U_>gOzkb%6uAk(dk6VFLS6w+yp@8@U1b*%R`3Q3vke9^$fyfq zolug$5Q(hWj32u8oB|(M=E3RfpDEu-gO#;xT-YzDjO-!#p@67Ef1}FU{V1kI|CZFBU-YCRpnh8_SK#a#z#W7WfN z$YWBgI<93YYMR>*hdXJ90)od>Xi|&X(S7lY)onS8b$tndQ2@ZBIxB>BhpU!afrXpS z9gAGTct0p7kk&I#76MKe&oo=?CsR-NzI6DZJ(oGN;XNaIyPs+k%`BvIESuSRh4+Pp zn2VvM-I@yl)h}CfAu{}NkA+#5{c(MU8@YrbEe$M=+4c_a?sP^Rhe*5>cqGF>5;gUHKNnY|M2nXb-5-V#gC(oeNj zk2LrC_|Vh@x{yKqxrKs} zXx3KCdhp+P^mY#sgR+$6En?jq=cIxZ5^ez|Wt>`H_Zj&tOJNq6-3ToT~vQg3mdy}Q! zMD;qXF2dUMD@pHYM1<{@4FDr8`Hu{EoMNdPE`Gm$-ibwK6Y;*CT|%yefh6yBFptHiRrC3f9eE7WvKeQ{mK^)xL;tX8 zFny3qKSPhuUEi(0`UDZliaKbh2C5L@-JmyfR{6G-Q2vrqOPjWIh)M??lg}$wwNR*b zRFT>e@qpWk+&X9?K;oO`3chF`vG5TE7#aquwi8G-W+n_iI0}c0T-WHeem|0BXz4sz z$Y5>!&avNF>hO3o)V*n(Tb3-Y8p;$(NeAiI^$)AVG$p7F?3GX8IDj$a(OVW@c)Pdy z3E-0VCB;khW#ms$MCHU6-CI0mxOYcJz_wzp%~ArgdT-8fmKN;)Kl2Z%MQd-Nv9!%g*Wz3G#9p6S8qCr5x>7)o0G`r7ndcOQBdP68>SHVZ`&Csvkb&5G;_b`N%jaTWtUI`2@<}f>U8sCTB`e0hv?Qd+==H@w z*+adB?_5OUq@8(uOz`QFb@ol@0UT7*Hn{B#+7HGtKC{SQx5l@?^0%M@MI9=4wCbfv?s^K?`uXbnzl6369u67b>!qv!)nsceb1JU`l3$#!bjb_k=hMm*CASoBd0a>dNhNZ zscna^4Kyp5`}eNmEg01fO~{?4kbcQ_(1h!?3HiDib3&8GW@A|)p?*l^| zZV0Pop^+b@*?{+3`@!!UZV7$r9Mh{p0*pPkVR5Il9IgQm@tf2Gj?i3^T}PNUN&va-E=iG~#hZ(kr&)%p0&noGR2+l7yc zIwtIg%f$H|?L!rwp^(G={?u${F$;wgY_}2+0H)BSh)~LTx<81uwQx{KR&jV4_0!ZMCEXUt@|(dH$eYG0#xg7$;>xohd!zWI(VD81_WBpMKVWZaTLiT*h|tiGp~ou`CqqC>f-<7`wm73uMJ zCFD2Z%cs-h0cR@&arB`d9GrKvSp&;vsGcM9c}#`N*LN3|dP>?Yk^iOQC$X5Ue<4R- zIz9~mRQx2y$r0`l2?S@Ge~4RAT5}!|7mA~6^40XQLka*5f;4y4-%{* zVy8)-3phw_84z+;|1{Ncnu+T${LuStec`oe_P$ubBuNC*m6|Clz0>$%%@fN+^NMY* z^QCd;o$hjpr9>F3QSZ-_fP_ER%`$haB_N)GJ6x`J#j!;%Rx?53d4P-Qua|mP@`tVY zC;M24A2(1=ON)czVN-jT%G&eW=;4C1A;PNT)5owyBn{G-d+`u(VaunOwx@VXjBgiI zT{DzA)NJLEoKAO@FB80%|Dxw^x_IC?b#iycdi&zJ(NN092G5@H@DBEb4m)!O6Uz29 zAjQMzsx`O=tH-q=A5MMoOFU;of?I(svBOE;z#7&!gyKm!QpmyM(G?gA(bM{@U_~U~ ztvOaWQ=|#hJ(jX_13IzXjXEyLh+>Lo0m~fns0J#X`Iis3|HWKDK%)ZHC(7f*E$Y8+jY~ei9X%%Qgm?^a73mj zGz8YSp37ex@k>W7#ZhY0wGuNZDpjP8kr%6d=OOBd=6CrH_v4(Qv5q7Fq5|h_=r$2w zYp=o%I{baX5<81GF$Kp!TXhyeNQB&1ciE-Lt_NF^YR+&Sl;?a37!5_N7=V*m1A&j( zCYQDUba8D%y5^n(W@*;kE|gxZQ3lM`Vp8dn15JyAPBGo z#`G`C7Km2$M0$YXkue8`NA7Cbtz+-OxY~E5>KEgtoqAO98X*($x39BkOQ|R~bic10 zDIb*G8KWN3^^nw!mv=j!RpfqsDE@ud>1?Q-u`jC>M)@ec^HE7;~bgX zcH&+xlyr#+ol)@0gEigA#kLS>LkR^9;BP?fDk6*b`!F&Z zFZ5It{_lzie39lG8y{~Nxm1|tfUQ_3aq^=YR9jL!FDI0SXnX6nGNC^aHyQAOqse$5 zsrZviL0D_lvN^qekw{A6{c7~Uljj6K{;pR`=UR$p(8j1w8Gql4o3a=a3g^%&F|m-b z{`6CBYPo)vzh0E;Q}O7QlcV_JT!uBd!sW#=%xLLmL#_L<43K$XA}PjCuS83>2xSis zrPgnIjT*UFRwbJR)$CaN96FP)$Mtqy7Y^#wEH)@|mfWH>er3&X4&+FNEUaP7hufT# zboxSq8hon$Mk*OpxL?wL3b4VtedRn2<#SHf+n^OccmK*x@ICG2#xm9m?doOb8K;OM z;JU?nHbvjv*dZn|L1WRgujgPA6xB-5oCV5hq&6cR`aO-^uUBFwX;jIYf?V8xt%C*P z02T-)0(X`cv0}nxARr)Be#}$*JNf%ysUG90C)aQ1%={Jus`u zbn9wNYS;C{3{qa>tE598NzP4IcY{ba9ZI-q0ed*JA&mM#DyyLl z@7#PBHDCJ`X=%4pTJ4ctnnzLDL_wv{xPMHWPI*Vb5BDE9DW8z0nAsKrvn>o}`@P26 zxDsmUVCeIsVI`cKcYt9U{Il2S+;%jx;?>c&iE@2ayUeC(Y(;eR%&wnb)bUltQB%`^mGt~ z!|MGwrUMXp2boh~_B{Q=YH|t-!p9IEVtc{(OO>UH(W;Ul#DHZOim;Cd{|oLfKtAU9 zS*+n6fTSO-7qRJWznD`tQCiN0eG)Q6nmfpVO56i)m88YR`|{iyJ$w|V*uO~&QFPGP zcWc%w3BQc;lRIgx4q}JZt^UT(kH~xGJ6CNbX10eUn7AE4cLWh%2?c!1mfnrkNId#l z``tR!rXW1EsB1=v=;48IHC-|x^#D!;5SB@*t(+s)k-P{!c0lt%+ik%IFDW zl8w;S1(zRw*{YKPkm|yQW@+fLI>)X7>;zXhh45d5sIj_Xr zEUs#(yCeV_u0i->G0loN?i*MEK3-bdm_Qnw_oS14Ea0sgps5-EDtT6s*8p>TYHR7M zjoLm}kxi9eX0zso@gY8?&rC&3!764fCGzjF_qfQ5S$B0hQTAl~gn8tP4o7G2I|z6y z>e~M5_Wxo$xX%+Iz@y-hvNZktxingbdpeHIB-ZMo?3LB+6!)7y_v~6O80}lKMa+*) zheBVz;i$QeeIi4_I9+Z$-8#KI4vHg6LY(-f-fX72UiHvtBmUX$9jg0!zj|KM7&rer z4^#lV(FJs1cLNn~mO^xIslw6Cgwq80EJ4`o^Bx`_bR7dI9U69T((M83!H)JJhfm#=Xp?Q@`P zE%$6BAiMGb24j!+_+vju(8Ym0PLjArtgG9yxr4BdDj`t{Q-3!k z_d6_x54FNf%?mW-Y~V(t30AB2$P3D7bqT_$s*(DUYN$kZF%;_$)1#)gR70t8IOnLJ z#`Kdup_BBfXB(M&JGHSc*}{p}y@A}vOUU%p3cVwt3!lmWDVPI)!I@?D;{D}>HVS$b zQ;Lt?6~>!4Nrk=G5;SazX8T;%Uc;i!pY7M4*B3x#-*U9N8^ZjNConV#N6Z^0&BeUXxp(q%bcH)*Vk?o=m*o-I~GO9c3G$o4$wN zQCIrq3J+Hzn}Av?>*08dl9z|p+gQ_fon*Kqqwl8W6SeW*)>$Z|=jS@^OHGXSr`^<(&-!;=NUhD)7V=no zV^2#(`1;O+b-jL3CGVEh)el`LNGLP;3_sfs(2Sj6=@~yxJ+8V6sxfXF?qKiR|4n)% zJd`YG8+*RKzi~cd-sdZR8;^$caQR^aC!?{*cSDa9c~EdwHZ#vF4)$A>tS^5Zj@6Ue z8nz?%1@Gd3O*?IZ3(95MGSA#+WAm1zaH8#tJAcH>7xIhEjOrN(Yw8iniL}$k^Q@AL|MW3v78D7k+%Zj|7wQ2ZQ zK@yMz(@kPsP%`H5fLG3-_cyPS5T5f0Qf{%`tg7Ubu4sHu=w=3afUF=>+r_dXMoa<} zLr3+txMBnKa>|?Ux$z`^dXp+`!f9TjPqrjCxW36ZEK>YJD?xYmVa)pEo378CIIm~l z9sHFfvegm#u_mAfN?N@c;wM>&kT-!ZS((aU+slZqqP_44eI1&%B&VEw!C}!NtDQ{8 zhh;bY0uhd7b%ld%z14fz=ZA@8vt_xfy*7-7L(U5pTAGx z!X=kA4%;<$UP)yQf)3Zx$@PV<#TBi1aj7~F2WBvE)*t^23-;H=xVJw;CXGvXGpO(m z=h0ev74pJ5%Sdm8ENNg@1T-Sbtp?hR(IzR7cvx%Sb$m zbZM;TYkg*tP?=Y+$IHHa?V;ins_VW|AZwj4_&gwXWBUA3*_a%A^<9pC7`S$r>ml@E zvW_=ZypgzuNgHQ9I=5551Q^)pou7aLN!&|SD1S{^GZhDvk zP1UsjXMRmOAnn(|Yx=(Y;eg@QSelM9(FbvX<6xOc5jv2km>0WV zebw1-Cp>Nau5H}BV7SP(f7ME`cGYH@11hqc^7%V+x*U zHxAum!*3jx9R_6z(JIM|=dZgXFm(@xkx*E2cCq*IfN5bQHmS#h#p(>yzFKBGA8t{N z>K*Fx!QI~Ag3@UJ$1GHjbuV58zO?u<_nH5?*nva9k=LAZeu|3nDDOe-d3d7tkT;S; zH4VQbPp8rm$_ATqNqvwd90wEahj<&O>dZz5(IgbNazyikL{k&eCI`)NIOG{CU}?0Ct(?Ro@X1 zL3Ej}aKv|P_VL?~1`WyDnHU`G{G4GQmB3JswY}FRA5`9q3Uf)$d`G%IOuM9F!95;7R9iSFCDJvUrTD@owSE1shf1;U5lN z&be~BxPLt`KE718{&A>MBT&)HwI+zD-!9a%jZsJnkH<;}gLgKI&sj?|^y_sb=I)^R z18os@F)S)zH87-u?CwjWXpc4cuKlJa<6fKhaOW_drqC7fn5Tx}`>`a2KZ)7+35!`B zXweOYXKJ%$r8B~V!AYTmZ&U(rZ77yD7S3*ahfN04I=76rDfLYcaA$M zy42ia2JZ;tqO1ZfU8I1bi;26{I91eO6I*r;QCGk>c(?Wh8#SBNKV{R1L#z%i-z>r^ zXZN4Cn1nvEW|u*+&`~VWVb#iIhO0lq?N2)NOYm#{1o)92P0Mdx9uL_!PsC32zvYx$ zF%&%NR)KkJb#2u&_R^oWs&JO|;$*9ZWF~o=ge{v2qSMosOpj;Zf9}8K;wqfqIeeVB zew)_!YX1mj)Ah(S(V)3aL(laKF}LqoK4Me>r1csYIHGT2Jr?udNuG+Xx3Tx+8&j^b znGjs;ed<}43u!k`*9$8;Kng~(-q>r^^2?{Y!8-~swbmqaWU#yLiD>m{--vyeehxac zJwV<0tFzjeGckz_jZh5#oIk12)y3&%0Q_<~OxDHIDMU#G zy1Sl5?Kcd-J`r;5=*F4Tbg58Sx@4#bjYh$RN%+*ccv4gX5ruAf9?Y>V!*oe&dG4cY zUaz+O2r}4s;V7l{N;ZujwG;zzGagI7bxt>L__=#J_oTwOr-KCIal%JCB5EZ1ep=-c zc5w|kY+eLbtGF-t-6>V+3OBl618o@WvZ};)|2)iCHSvL?8fUVKba1o6O}?K3f{W&3 zKdf2|da;qt|In<1Q2Rk&MZPG?Z)}=-_-B^Wh+yoZ^oo<^wv#)(Cei?nb)Ju$VTskT z4}y#{CpAlFq#7#akev%WRcQN*fjPoIwY0SCt>+e2Pz@!1nh#PHe&z=epl)wJb5kKGwq$?RVT6vtGSL)^bbXM0ME2xNKb3m!4}Zd38q@mT6C| zQ)_i)ykt^??k%XnZj3p-gQz#pf_7h(rng??_JrG3#>$s7!b_pE>wZ=bIWH47o}p{e zF<%T0AivUovp35La_ewru#cU2YKFI6bd!<_Z|S5wxtnNpvcfPKV3m6VpjMrux8~^2 z!UEvZMz71jty2Idy6UcYJ9}TWK(n+p0Pgm^ek8W-WNlbU{+FQ#%>?H7$WWwgx@r&6 z6214O?p6no#fg>`d1orlsL_SnX!rOuWu%g-o`LGY&~Xl_pqr`NLUSaFEJFNFMsd!9!XnP9ocmxxAB?I zts=jImSLNNfHD0}udAz2&4cZJaKCbQFCR_mHU)=-ysk6jikh}3iDGz?)&^hR#x$RT zF*uAK!KsDfQTz7Z6{dR&Ov@h&PX9!%b-ew^?9cTB8f_;@M?)UjV@l+!VEoi}IB{iV zMRWEdOCh1URvwNLq5Ju6ae*jMq6Myesw8ZzDnS!A0IQ_^@ydK)Op zq#!a@=9YaH9*P{EoBbC3tA8gVKG|T1J`mZ{yP0(vywqtE-RX`vUn(ufV(tM^AJ-QV znuyq?7-=s(6Ai-zWNl+X`mg=`N@d6OKcN+Kbfr-F!B6DEiIS)t{jT1~`?+M}h2@V$ zS*;=HvTN#k&QsFB&i&(Bj)oVr8=X7G#>Ssdy1iE)@&k`B=mzfe{Yh(7ifeWI zbj}nZSI^p*AtZX!3^uLMMQq7{-FA{atAuk#ejhI5HfJk&K^x{#o!;s-e!WT7ESY%& zFJQ<5xy)yZKW2sB33DMNLom^?z8LPT$SIRZ?6{gnP&T7i92^!B{qAUgdBI?iOt)FAkHqIr-)#7-5B?T-5VD3x)N)Z zBmgR)xvuy0NAF%9a%W62%@w5!PKSGz&^-lg6 z0TiW@nNj^o_fR{lLe$*bb8MOw_E-`{zJ!+0kxtG|{iq>D*y~P@HMPr=u2OFn?J=v# z&7*~00kz(SwH1&QJ)JH#o|b|wcH521>PHwDN_Co%b(1(MHn&}mon$8_xMWcz3CThZ zzK4=c8s_*(&Ob!Fys=6+@0RW#<0|#J#K;d5gVW!vS{fk_?7Ud=+I3^dSX;La9=o~Z z&G|H%6TM~;*Zs*$_aH>;+qFFx$Xw4HrvtD~XYnH)$W2^>7bi^h!p(+$P0)^4voHix zL24eaGe?o%G3osF{!1D^8`IFA|a6|)-)>kpz{q|{a+jay> z@4~Nc>a1yd(`w&z!xlX`dcN9^ZV=h=pYrTD-0N3EPRvc}#$9g(?gilSAoHe+x;D=) zNs+=vh>aj^YRk(ihwEtkSoD#OgT7mCGa`e#Nt%WWoE+f*Xv(}8Bq-_w6U@14F`tM< zGGSXuku}EfxGQ>dBs?nd0?ALK9KRhttFL7;(;zRbrNeqxLtU$HV#Tug1)vI7gYkSo zMn)=*t=sM|C&O#iDweN5_6kQP|Er+sS;t=g2(rsUNs!6gdq=kFB8qyFri8~v@bL4L z>n^84Q6*2_@QT2g;Mf9E@I69)1<;vz;lt-^m(BzA*8C zR@)uC4A|k~w+$gmVFjlqV%9;Q?2Tqh-0W&}_ZFii7`qlCM4y&Fd9?eZ)O@4De{mKG zB`f6TsWg|UQyr<{HsW4b<Q3vYHHT?qamgjYvA(T_M&a0kA7>-@eKDXyr|ntw{;0R#{t6wo zc{s;JO<8BNPVa$BA|Ga&>~&@q2Xo<=GgSTya5!xzV}4O-ru_%a-U<8(6>bvwpM>tl zd5+m?mNQ$ftjXLEpzf?y6FWVoY|cpUXDmN704muZybBecuGvfnsI8SlHGZ}8S(1x$ zyekrgpjjv-+8r_x3`{Xwrksi?S@;7_p^tbyhPRy*+dwt81Jn#6y0*@ZyZWd<(QSha*FFyW_FA_JBPHO|TrNbZ zC>FQzJ^Tdx;3~LYS2==bVt4PgNoGHBRLKWqt=G`AI3)Lbub3&PT`GqMRzMs|@KXm( zoG>jgvwK(btsy8-5WE#bDsDdVyE~5O#_z2`c7F;16Qhk+WIy@0TM-kYZ}|5`6@}(~ zbg9xL%FDIh2AZ|?-k*qz-q&8X5xiJ5bgmMr$a$Mf6U}pLsq=j%(H8u4X#G(HWER6`!0l4RWNU!~!hMHmF ze(*v!8{My(*48lzbHqj&~ZAY0f!T4rQQku}ur|fr` zzu2#*x|Za#Bt}J2zBqd&6}4{EDuF)tye>7jb9Z2fT;Wz6u{!DZE{6IlgyT7gD<;^b z)OlTYkU8TKCRNduf8f?5J+i`^rs4_v?pZK~%8T+u$FVbsI(|B?brH67O?tH3pvVoN z_&yK71c5u7Nz8t;b0~$@HrNJM_Vmsx#X`fIbi$tBim-4=H65w2BPbda63#Pav*#*qK30d@1J+6q639H)-qkT8ZJMLKR2U&Rh8Qe1KfD0ir3`*bh0DxjIcTBuj&9qfPn-buQ zsdOhH-m+3}yRUFg6W%OzzbJ4+)@|14lDV=MDV+a2+qzy<&KpBIXQ;XJBZ8bw=B z%FntQy7V_%UTyZ$W*0ha9Yh+>57#ZuAg|f7zXA`Tmpus57l1*{Ty0Ay6AlXJbIkk& z3$DU{j5cZm(4ZKmLJBr&&lxtS*9Lfj^d_1Ai@p~dO{{m#yWW>9eZ5S3eME^JA*aMv z!B)3v?i)2F!XFDB@p-5|7FgeREc5BX*!#i;H!*W{5kOiI9Rt;olq_Z`t%0O3Th9S< zQ7)9fNU~m-UL)qpAY|(D*!6rBWy$qZ# zqxr0_7e_xE=SkMa?vEuV#qP&bd#N*zeAiqK(f-j9QUlLJ?ckI`EdU_$dPZL^hZZajDgc3O|9F8fp=H0kY3uGm5*6<3 z6^^y=*Z?3@^-ENOsX^C3nXQ`wvxZ?{C&dMyeE{S!y-xoxX57Hc06yYB*(=BamU#QY ztCQP+c?iSs#Rwo{pKr{&tx(#jX}DaJ@>-ZHfR_3g~N|z+7Ntl3mxCcXezVd&$dw?p~d1 zH^1_RVWk7YZt57}air3u{Co7faQd)&3{j3KOzNvWq%u1_J(SKD3$@1F+3AnDf z*nskS6W-?k)e1IOP$7{~9|_@)JYM9NXo<+$Sqa8R6eMw}QWh@Fr} zbdN=U&Jv6NB5Z2IeTGnTu17gbDtvuddukGN@YnxqZlH?X_}WvGzL@?v)1y@@waVyW<`(Ucylw-tU*z^VtVV16$1sW z(H_8>3SJVJ8wi0@Z6w*8p~y3)*VxOFGHiW4FHn83|6ONDM7__rLEf z_nR`2DeD~?pV<-DZ4sc@uXC;l$GnedZs9kcu9KQPaaLmr9wGgJ8RPS)VcU}E(>Q6I zZZduJOX6WOW`!S>T^$LxsEP{jR_}Fkto+#=0K9IW$||5E$H1FE|5hL@t;?~5a+hFM zYR^u*{eI&eSCJyEXSN^!6fG-3<*Nk`a!aS(^c4_E5Aen!RW$d>lBf_$>UQ4<4hVD8 z*Zt9YGw_M%@JdPWU&)m}BRNvWnPtyrEsTHd$CE%Q76AE~GU0d^wl^89)INpOk6=%5 z<1=OTIS>chJP=vMK;rdzE-l9NByRTsm{(G;QW}pDZ_wSq{=?(>*J?` z3esA+PL4C}DOE=$SqhzW{uD3$!#dKOSny0ardAV?lpIAFYJaV8+E)#yIO{)h*3?NU zEPblkrWLS6IgDz~2krA>l>q@7-fq}MR-|!D$&LIGxLd+a&kly8IxLxgce~vbrI|Fa zRx0WdhsLPy1IZ45n>w{Wj^jz@6y6E0G${AHB%B@{xW{eUb83Ld>)#o;f7$&_Az{>B zSI5$3jFinPmp*^>()n&3qBonodZ%gHTThv8cXiYPd;EKV>to>u52D*CiR!1^YG#g* zncK7M9>eE-r^ms{8rtm5(lICm5qYu2`0N`E`allpO6I=5r7vImOiJu6UK<4|OL`a}$8Ohc8Z#Bo+QgYu_2x)DyiK zKtQP~0xH#tbfrp1iXcU50O^SI-fI9sKtTwKNGBjY2uLUNBGP*&^dh}OC?R2Q;_v_L z?$>?xe$gk)y)$#>oSAdpnRySu81P!dY&UaQbM4~Gv&cVdF`J6+X`Rt>D$eV^^Y3Sj zwK!Rf9@l-|OC$=cZI)RXrc14K^*+IouuKPV{GHMCvGtM23sZ1h zwXN+|e4qPGs#M1oujMa`LgR0E8>+igszh_mYe%OaJ5!i=XBZ`P3p*sGl547sR2zFY zKaAnIW_xwAQ)%qQp&qR7WTMY*s?An!kNjizZ|fHyQGa)@KCf78o1EW%|9yieG9alT9&zdT=DC8?F#0)d%F$BLtQoC+R@MYU zo=3oHIK>q*^WNGy&OQYpuSZaXKKhyIXtnwyuob;q(}Gn!ZZe-D;09Cs4h((A_6ojp z?|a7~fjn=T(VapbBg|#Zm?~Z7Qsc=%6jy*13%JsvUAKcyCP}Zo?*{hL`NLWh2-eJuoQ9Y2vcHZ6Jw~@YjfeAl zu3U$SvG~eUU21kS7Wvi{#f98`X?vQR89!3?xJT>5zkYZ7vwIn)zT443 zBJC0;miBc{Lc}Y6d;|t$VYkR+Mv6ftTEt=eTR7RzZZ6p@u0X zgSl^yJRGY6Ln4YJ9K>FOx}FAtKFK|LhdY;T`gqz-OxHmidadqAvzt)Rmz`(U;kLV9 z|9X$tCKt!>Y}cr0e37y5K@zAYr@gDO4#VzeU+=n3Xu!D0SZimZ0&Udga!4ettsi`? z#A2X3%%_=O-=KjURC37GA0ks0L>}EZ8k#-y`umGF%^v;vJna%49BV(=vhcTZf=u}c zTC;7&&aiVMi1Y5Bg-!X2t~zAVbL#N-JNkBVJHJYQc7ILE_hLJx zd^{4E+^%o9RA9MKw++!MqKq}%+~>|OD=Jd?*qL7k?tkR%v)^XeIyek2x{TT!uw(o1 znW>i!X(IiW_^tt#@&N?knPG2FG?1gG8*~kj;5=?J(%tfW%<1WPV@6}m@IF0eO9%Zt z)!_FHv8s==3cCejoRrkaKJFjx3hUaPZGGH};yNb%lg*-Tk-2jza6<#ni2OU`oWJ@am#j!R?kXZ`?dnN$_j@#dq065?HVI@^qOa~%Mm-^ zYMm3emiR&UZZrY6{526gPu5s`2~E(JzD%4_joE714;sF?Z50t;DbeD#VR^+nMe260 zCfdbb3ClIB+-#*@bM1tVxX7HKrm6shw${Tho0QdV0k z=MHe#=W98c>MFrVMpfT5qrSv#96_I3q(8jXHWR8<;uiTNS9CBj^Y>uod0`nsKE#7ooM&IY?=?AD(NA&uTE{m*3CpBPbdCTuH#l zu7zjF+n{suxEKk*50ze_tF}O+ex(OKC&;8TU+cV(P#MV8x+4O~C{A&$tRFM?!XT)i zo^%BoZdi=w=8>}JLT4GzcdN9zemcmS+Y0?HB)Z!0K2+-DP5ABuRLS|plu0CHiwN4k zQ=TV{cA0_=x2qxp!$!u(xA+dIFquVsTp?Fkhk-+cnl}N{qQcZJx7(A?Q29RQLpd!2 z^nYdx7nkKJ>m&SdfJW|jgw-=vHpQ`Y1KvKefGb}>=QQ`;od9UFvr5uVY@^heX+sz7 z!`XE!!-uU3-?|?>4jM6cSxqp2X7-C*CXI=pjwg)B#RT%)&z+T^Nb_)OJC`t$aDBb` zqpB2i`F26Yiul+y(Kr<6k%Z^y_|VyTs@Zr+aRKD|5v*64sB||#^2Ispzg44W(DO!8 z^z+EUHAdL62RF@dJI|9$_p23Y*G3!NbYJ`^6Gd%R8OMk?HNLBYl6++fO#;O{`4F=Y zfJxWwiZNyH+IuU4v6&dY`Zep)DQlL3`X)7};f9!K<6$&ylJl>6bq%zKuS-kM#Ojdw z!MN0xH`)`W7v2zF+5Dq{dKJ~j9Qk{ybcB~J`h!t)XSNqx$yGp!A+; z@=@7V;&C*O|2VqUt0Lq0k7g0rsWh)H?^ie=zA;h8Z?;}Jq2~-rDRBLw`t3)WU7BMQ$%m% z$=O;=K=DMTV+Qx3o&~nQU3P(!Jk5z`ixwP;tdIQ9HR}z1?LM*AS8r@?J|J{o{6DY* zhLX9_>H7}5EKcj3zACvrVB@v76Vj>8mxG%#9vS2F42>=!??L{HP_7?j5o`NIr>!$l*j2t=9^E?p;w7Mu5_INuVzQqpZmyL659KU^ey;d?(w zF5Z=V+Vd)K@=VLha~ZI`_vPPT;k%kPbmQI}qE6jXN}sd&B|*%)S^r=9(P*9MLcN#x z7ayyh;HYQ-j)fzxOMr2NkhR`I=IX1{tLaq$MTl~PyHZiznR*4llE)&U#k;cP@7jd` zrZRru0{1Jg5wWrGE7BX2fXZgFxzgR}xU$YD1K=8KoHeYy)%%VH*RGk)_XZ-GUqhYy zz+B?7JHQ?EM@&WPlAW#o?9vf~KHAe6A4NVjJ?xXtQEbN%3`v`yQP9ZWCO3dX5F`m< z&`l}F@32{;`-n5Hltke58}aK3@49PzfV=jZpV9;-+i3{k$Yc!Soi^A@GkWr)9FBo>(fp zxa5Ye9|W7wb`ykKbQXAlfg33pxy!?xI z=>!jJX?F#?E6-FtbE#O}9B}(9{~LwUBnDItD{wOU*k}EV?;X)c$~dZ!15%A#qKCm$ z2;<{`SjLk=aS>FqPjF8Q8|PvUjZh#~=K(lInrvXBy;bd9F4eml?VleW`U9LORATqq zsff4_UfK(0rFCTY6CE*OgpYeX;)Z@nvc}J8DV-){047~^u~OwxiV(d|8X#c;dYB97 znWD3D2NfqFD2@ZKz$8-u>5b2V+HvW;EMU3Iv7f^KPS}BFG7krxjF_K7VjtfT?!?g( z@%nA|UmPWV2Hl()(xd;v5U9;=a_UJS_LlWpMNhuwqw&ixXW9VPJ&9M-PPvc{pWSnd zoq{M$o9JQ^=ziCLgg;-0q17rP+xeG5U5^T}p2y!g#q@jklfR!@w1=hJzoCl(ZU$1F zDMhPH^(*XM1#pj<%4M3yiyd7-*D1k}iFLwPFWKBBFDC3oR;CFFf-1iX;-nGNBk_N&aHtMD!LgEd_VImUcG`KrckG zf8}eNLjQ=ej+-3MsmB)}$MRQh+!p~Fus|-E5avN*n?F%r-CGKF#7#}Zv@4Z72i+pN zv}7Rn$BA?~0A>XO*eQU!>LpH>)ruFi_hb}JgKSQ#u}hYQV9l_i0=GXn(l9q9Be@3-sQ1O zW{UHhPs`Nz$4PFg5?uz+V83g)RbF!5X8K>D{>D{vBd}6B0BGx)=gf{R@ksu z`^GhiUNa0qgp&)xB z<`_W1I;&qXi=3tSPug8_5d_I8YU8F6faYxy2u;=R%Jun;^SYTheCz7^MZvrjm*IE% zE(X!p=cc@1N=^G!T08Yj_@?#8j+{pUlO^|_74xQ-R0<9|?N?Ka+&U z&f7PREw~9DU{DLgGpNnv+*?)~lj3Q%l8omSpv0X312`IiydfZv1HKFC24g?pJXnPV zHuR*Pe7RzG*T4xT|oRIRP6+fA!$cn_dj`{Fh0qi|sK#ueDM z?3SgLcqiL_cDRa!**wp`_Xv0;9mi>gBQf=^1Ly&l0rt2GlLCOnPY=h#2oOwG&e_eM zo-RSV0hY&ijpImO#PUrBbwEI@W&heF>(q#;1Ykvy%#uQu|(kDCQzm9 zx_FeXp-U(ebD{hDJCw||{3A7NI8sJ*O$j;jxoibcOH2TcC>6x_nIt;iB*15rkpJ$5 z#OZvfBh@>fvRK-pqls5hKK-dvN_F@Z94vufbSs&uVh(> zO7fm3l*}I3w)bSr$hQFi5EgWVf1th+({$;yI~!5}-wl9X5>6y` zjk=`}dVcWD&?=N02#g?Hg>LY(T5t1*vKjimT%-6N`{JR8@kDmu5z4F`t_XW{sQ( z$c>P2<3Y~~!VPHi+TDLeG`uUl_ZL49Tdvtmh_y|zls-IlVtZkeZ`0Vdejd=cEkv<< zl%h{0AY3b51iA2zz4mHOVI<*MPCbmnfRs_(nuPe02{JnLc4`{|XvPS>FRnqmhj~pZ zBS@FYQi~&>o}eEa)HZWVfbFRBR3q%>?KBiykcSQVY` z`0}A2$lB;H&;-ak-GrU7cDoYUvZv9!gx+o3g&$vKDNZ|=ZaA6sGu!PUBBe?|IyKIK zlg`ykud`E7IGO)R`1+lWAIOBF^&`Jk*DQ##Rf!LS5EQEUrFt(b-WiMCZ3@d9JQ*=@ zmn73KyCbRe>g&JyI)ENGSny7h?##|;Ls5b6DolEbXLIyfh>`lS|PCHyMz;$Y*omNsmyMvPXz-6Z21Nc0JL%*_Y5kzwt{e z_hUqs|J*qMsS|)iKC%P!v_~;KdLEpwTwP$FIjpDcf+IV-;o_zAb@AW1o}IOrP*7jx znHm%$tEc1Ef0v^M+i6iyE!N?cQKf$DzhywiSml7Ed)lh3pl zU}DDCg^Wx4H@#wCT4YO|IA5ZsTQO{ROq4x$1a!BSYUE{AbQi4q*~Fzb@A~cVP>cl7 zZmeHy1q|OIY8_|KqU_6j$OkqU%_L(vPN{T3>*g_>UEaDx|vWD7fL<733Ltpx0pDy1S2iMqU@Iuu)7zukj!u zKioS-z~*Z~wbou<{@BK$-bby#!%w@^>-5=!Y&KwORyAVL-!-v3(@F}&7l$rSGRg>MEo9#K_ zxr_anQo3K);MgO58g16c1u}J6>#I|eEs%TfSPoZLp^#3;-9F@O(P^mP&{~@}`_<>i zX&3iJu!cvMG(|&0lB?1AT?(}hx$9d-{yZVUfi$`h_(|cV3E8ZdyC}foj=<5x1~JVH0Z%zNV(enXEEc5e(nyI=KG;(yPYbuo$9vS<5m`7*Mmo*78CGCI$1JcK6#kZ|pY4LcYrx)1&MtWSaL5qbv5DX=B!T z;52nox^G}xkTO$B5G!fPRbcv;$Rc+O{R|gC{2(TBUsx;9S5aSBWN~t&DloeECWR?d zSEt{L4{j&1t${%F_)2Xyaq8@9ploAx^JUY4!^taq9S&DkR zj9ZjxkR-_$W5Hl-T-k3#BZ2}Q%asYAFhc&++}b_tAmm5%t99!AF8X)I4sb-E@ zx2IPbe|_8l*;NMCx@$Pb@1t*&?Cu)OMlpYiOW~5X3nMbaHzr=y)jXjEv%ve~TYf7G z)!iJA-+V@ z$dIlr@1V87U5*~jQ(9EWXYhyG{N^OG!u_YBf35w=P)s_ZCtsz!z=`Pmp@5CDmUa~5 z&!>(joR-LaG5uFXydRZ@egZ1{c>iRZL+Fspofvcr&`7{2@6}n4%UAK_ooubItkU@;(cXkefxs4HXyV)o@VYBvgSJ_ymV|Ln>lQT`+Igk412@KDaWa%oXHRC zq)HS-_19wgpIp?0z;c?B?+)s@=*uFWKZ}1xOgCzB)NA|Yt2%6#X7(emv|aWVmB3bm znEjitfTT_C2Lah`=>1oslfN>mgf}o6suzf=LlLu|xw&^_p3RVNds~?4L9+5kHae|& zr)NS7a$YL<1|m2fNEJR?Bch|0Xmv}M(tCMEY&o*(a`?A*r^G@?e2kF)$cKyKY>H(l zioV9?Vu1dJ&}mJU4&@++2m@=4{96Ikl}J?tIAb*F1vGcxBE*-|_|;AAnl|t)PmYHa z=hG1ldI-kNI3#ChpAk3L7P{8Z*wHnP4(0;W{ROWq@YgMfa`%(ofFOBAzG9M%5M6Lh z5U4aetFW+e2JjP@Yab_s%`pG0J_oy%)#~XIcd>$*NuUKotD0FG>JsB1E?I?LL}`v3Y1~JN zM9bUTt$%e&s9GdLKR9lrnVx(5mL3rIUsru8A(II^SN<0&J z2Cn^*M3v~R|?q{VEG*QS}6?eaP>XlqrH zN<5?u3PYD7*912%Js+)gk1=x%0b6otNYdddxm;=Ah1ebf**Uq5`6O@u{8v2*AUd2w27yz-uxj0;k zHA1&eD(gJxuWTSLifWLf2L5rCt|QGV@ZZYG)fKI89NMP--nSny`kmWA1q_ZOzb7B( zM8=Y68}%G4Y0FUwH;_Q3^=>D0I(DH2{-`k29B(j)YXZb0V@ntrQ)TR z0Fwc28Mcv0#tsBh-@^Ij9Rycm5Sa+X!%yEhhX|gXmy`?(3XFDW-SbFZUqKqpfOyJpmrQ<@Qts$r^VkRPy7kQ0TdnEpxpN zaKPpU9(0gbZO%Lw=kPv)Fhlm9r#!uVM+Vnh^&VUcqz5BTHnd>vg$7l8@^o+Ta!~*L z?=OHSLH6EX@Gf~zdhsAG0=-3pYlgbO;;^Y;+xeswESrVpoER71NSELPA7b9wRosBW z`On3FH2!->6#ve^Epd2P5AdfnKQgbY%wO!*aDg}(EddY}Jy8UF6l3GGUriJc<}cKX zn0uIV`(U7^m|}cGUVH-H8_2;6$0rmQgQdsdLgAp`boMc*6xRFpgmpv(2lMFajSgM1 z``uk#DsFCN=u~MS1I_XST=g;IdZr|8LAY00VK@jPs4iRGaug0;AmraTtoGQ{k-2k= zOSM~Pe&;ed7Z=HOkm4du*4^J|@zN%Wh->&xHsIyyBir1)*cM4cmvGPfMad6*X?h|G zAi9elb{~$2Aaesk>DoO&4#OiOa;G1EWH)I95;bfG5q7Q*=hnu|oKmJ5qA; zaV`kXY4d&e{toL6d__gL2^S>t3c}po-QB9hbL=zy%PfSQn_Cd{H4|Tv8yWtrstWBoIr|AK}07;(g~2`A9=*vAI=^=1kvdVB-5Y{D5Vhff!2m z^5Vyx!_|JT0)sMZk{BO+Y>vT))kh-7K0ovFuA+x+{;54lsoIMyeWug=34E!^mpBs} zu)5`q%P4reYgxI8(a zdH6qtP}jqMwg?aNtqB4md3-_0j}r+PFm)0#wNh-1ZE@2a;KQy_Nw5M20?qGV+OC>& z)G5?oX;GM4{UUD})DrhoD(epczV;nlnmo`@&b$U@{w@Q<%r4nIAXMua$=9L`TFl^O z9`U?4%c4m;0L(EixR#_FkTKj6c;b8=7}@tkL5uSEMgsUT&{R$T_jmR=4a%A*K*hl1 z@+~e7Z;B7`Ako(;gj;Tb@6r37o za+6j1g#6-SHaXD1BjRXPRaG-yMdGjed5@ON5+D+qT)yk!zZN309orEaki#DPWn80lx5r zLZz7mlpeGmY(jqsH&vm%e-3El)!)2NWnl`S#DQUK%-vYW+HE1POJLr4MnwQp2jNi2 zbm{fCCc;K_xnwMY+?w>~lKvw{ohNgFb6{SVL26bSC1qtJ!K~eT1um~lqO)wNGfEVS zZ{(N}fz)r~^yP_q(o61*h1Dor_VFo@x~3+;4ivK~;F+&m2(Okm zpRj^KEP?3^i<^a(Q`x$(mRMg@DFNTY8kUc2S5lBlQC?xQ~B>FU6BV);{;gIRD*C_y4iGy13XKlX>wAB|sJl{0Aw8b98 zH&&BFKx@?JuFHON=(IEJNu%2QRgm<1+=<@wt44vC;~f$ZCe-CBC}{B+4!goV(%b!hZyd>pUxEA$@;xSMF+UM8_8MYA`uJLlRZeTelt( zXkpQTOODM?Ol`jZmJd<-a~no~my@ucPyk#^cX|q>f(sJJ=jaEE^T0Pe4U7bYnI+!M zIywrphJAjz_Qq}%13anP<{)$9SpZBun=rk`)SfzQhf zTS>Wz-ELQ&E_Y@sKvfw1CGrap9s3#HEtV4olW=(aZ8rI-prKX@L$YlzF+*O!$=`j; zE(v~7Ri<$tfe8oA)GAb=+i7tbENfapO$tgaC21x#M{{j8-)0+h5P}UG6}`s+-i+8W zv`4pqgKHra%#HyEnQedwmW6SHI`q_Hh=}JF^5lSA=oX_>&gCzzl}fTTou5gQHeIM) z2XBN;Yx6g=nH&eZ-bA^b?TKU?ta8eolm2Hkf7F2VCah>YK6uytMO{mneW{I|#ZW*@ z8a48kzT;0xQ%`%JVI#TC5|0;PHNI|oaN;C&d@ZbfMtn9bydOEPsy30c6IJh-{AfqW z?4q7WeI%d);{B6F#cRR*8&7pbp-d~B&4dr_$>`R+wD!eZ)o#HM>3ygTts)qHW%O+t(f@FDzg4pdB784TtL4rY@QcSJRo zj)Kwla~C-Xvyq7p+p`WLi#+8X3-x`*;(lkfyeHLjsB z^c54)!Ohy!XeW`ouVcqtz=a9$MvF=*Y7k<3M~tnuWUZ_W_5tc)OPW!6!;|Y%W!@e) zJ1U-;x;_86@r|Rc$z1AQjkjH7jnuXac%$%3z_+PCx?r>?oudKTcSa~me4e8VaDq^{ znRI}NtWUeM%K1!toVt_^)7f_gESzX;_-K-&q;S8Dm4up{SqfG@WVIF(y$a5+{#(+^ z4jY%pMsriT}7MS?iL^KTHl-mx4CSzPt?zRWOf*=UObLYE9QWk!I+S z!@rMA3^|vNZB2l;_4(}m|Bhy%H5smkLXGPCDm+d-6b!@|=(QO~6ljN<3@^@XjGNd# z^8c2TSs;Z3whLVw?cg4K*uy=o!uk*WOO|k!_c9p)&@y@z_o_G0Z(-cGFItQmn0|M) zfgLHd3P_3H-z61zowY%`PCpmX_`2}nk!Y0p@e5_L;}RolBE>n&x8}zMmTf;cP5@8M z!1@ahqFZ%=r&|g33ecR7IJesO1$~8nHWHfZ;ivhgSpG$>u}^yVQ9@YN z=%>GLTB0?K#fG;FBSgkP``B?xXFV8XB69TFEfab9VQ6`nUW{({%_^$abaEsMMoNDv zeb2`0a#Q#Bm|iJ1-)NzX1u|VC{oPmgvD;V!&t$@UNT+u3F#WTDq?2!Y7NX{W>f@85 z1{n`=6gIHD`z9lpA81}oj+Tq8{i#>ZW4I!!r44rfW9+T~&E8xOaPut z*Nl8VLQ3BEMA5&zE|VU)#S&NB?z#1oiu1{FEowCvxH>PjDKaP4%i=U9o1bf9*GSJB zFvtcY1r7zt=ifTb=P)q&?C-rH*K!EBtV}fj;M-2wX+M9s?%eL;k_)!1~O~+B#A$oOvMFmADQhY1~yyBlncX{R_J_w(eEa~ zy)%wkt|5KN1UwFrzxxf)6Ag%rZiu^A^w+J!0U4oux-xU+9E z*J2mxxN+<=4TG@32aU0=;)zci%%3GMp~%0j*-qJR&(oJ^##J`lo-~8zu8#(o=qFe% zo_1=>00s!fIV>GN1|ikeFKXI{;WT9YdYh&|}(r z+T_!K542^D3n@ULJGl{#V4g0N(df>st_J64Z4-rx5NCo64CEYR!G4k*`s+xvQBU>A zW3X%8Lv6>s@X;bVTK5AHfXIM{#>U1R0nh7!+BZz^w3OGcJEC>JLAvJsP6n!N{il^V?2s? zT^4X_RV>P(M~(sT9|t>Cr}C-s&50>$T+t)oKsEp%O|(^#=U}5Ga`x9=5P=knw!3w7 zEsFMm0Cky}E`tEX+Uibl;dyArZ{Km1e{zNZJ$V}D zJs-q&O8S}jtok7vY2$4k7zV%A)#a^KSdBSmd{V7vCOfwGo1bNpSF~_KWu*@Kkb?ho zD#M7`=^%?#G&MGQ(zI$2q1G4}eTUE6yG7*8Eg#|#e>Sfjd4|d>u?(JxcLeUjFD&|a zInr!7j8r)afDWM8ziI&^)9d>#Ro^`CS`B6I+gY^9TH?H+KUT(qx|^xylsF&gr0h@2 zY#LxT9eA_hlcQhf)aeyqb9lPC?ROTM)r@@Xmr=(%M5BpREqR|VQ@eiB>tZ(75MMRL zy8FwoVU)xN+|Nx3L+>rodF{{omYCaC1b1kN!CZ#xjwi4~OV{Y_V{LJc_0B3&t#HCN%23?P5yd}I zzQB6#VNAH>uZ5+q2+~+d;lTie`q{*HV~j(Agq$Cmc32Cm%p4_#tzL5E_AvRd=6@AP zW!NEsi)LA6D|ZcdTSP-W849QqAY##pz=C6_N1q*OMI4-|5KUQSP~vT*m%4`8$}Q;a zlHZ)}Cqs`(qUwC+P5vV0#xL6a=><6CMm8&8U%wKVvEIo&970$lyG$f^qYHyH;v%pq zw+s~`g9N62*yS9$;Fj+h0i(R(^Hc6h5-s(9~N153T-j9WKF(8XWQG>_rx zqasQ|D+yPt=iiS)y|$js=(+E>pC+Jzn?xdalL1}b$-qSw6#3-7&_X|3E4{eP$OW%t z=!dw^u4~89+jKp}&RQ83C}@|e0W$PM*9$PCtyGduUPPMBNgSe`0m|(9e)ji}Un|+4 z!3XUVO}k+FD-Sc%hfEUe47PP?RnaDs8^KpXzi6IK{B;&RE)CF^{=W z32}k5zW@Z*S{L<&<*O@FM8x01lc^Bti>*+#3Cye+XqkoOK;B`x=1qydyV0DXXmRNzE{6`<0R5Ku(Ruw7gk5(3PMEQS5YwV+L&QyF*-@G57#uI9@EbO<%H79E@W`k_R4?N+F-K#+e^*rz~$pMrwV zhGs-6Oj<9NGgtbnO`9951RIl_z9PK8j8?ywDRtfV)w;~BFJcOz>pmDA*#Az|pem#oq>hQL?>k`=tN+dHOJqVuaYKX3=cAZS2sdzEAgFfF~-p4{fi^yobn- z_!7BR_Co!+{o1ATiEKE97d?hdGjpLSbOKxcq_=@-Hz_Pf7U$#ETT5*aD%JDU#K?`t z@*>s~UG&QdrOhHpyb9Qwb)GBf$9xHOao<5-9)`X~ z54r!zkF~7MA4dvbnw(4|vQixHZdX%27ObD1(@s2_DbRaHFTHbMB6NCvK=*Nx;q+jH z*^n24%UR!^=Px&W3?#Kw^T4&?-MV-66$F=$z)+_dd6F=r+qd3p`9#7H9 zBH@J}fM0)(m1vjT$&MLv5=OTx&zLtTF@ivbOt@2N9WkMYoIPC;t$)1?#;d{R9vZWz zyv|FJ13kXppK70u$=BrYLCfh!SyDv@ZJOh?-QWLY4}(jOM2yVUAHVim0Dix{N*|$4 zJSgHPd2Y0tOLpG}Sg^VDP>nbH);kO;QVDx8sIw^KSWS?$R+Czq?ap(yT9eD!yU82V z0Y|f#eaUik%0&q2&2xFeCln2L(P0|qGxbjEzkV!cWFgrKbdCxi2{;P;2BL-H^$01_ z<1W=vx(AX%)RgdUz{CHGgJ)yzk&#ri97U&&nyo|?fIc5nsQYid5{Wd!^QXCOkP?x}7YG)L8G z7A4mrvOb-;nkE0x-K==&=u}45@fy6WKB|P+ zPCSZcf3LslyE<(}L&rz0<9BMO+x;oq;>;+49smr}#hv8}qDaEF@_A`p1VWi1> z1x6@!ta)UX(Wqy9{ipW!x=eiuYBx1HlYMR~mo5-7a1Px&E@RL{`7qRy#p7i@b&8`i!Z ze_dbv&v5+B{B!p6v6~y51)&ZX@el4QXvd}6E+lD}*ZU$jrg1@KRQHcLhG=H8zS!Al zz(t_NI~%Mswy9}4$G_PpHeXf8_w5f&K2BXi9ZiFlg+je>;e(Wky%fH?3dm8yn)myZQ+t(Tsh{eT-+#myqX}5a&Cg%_GyOB^*`SdFucsW``)GJ z^vS(bCYX`1EM=o^vPP-YldbT0dY22zP;i%BoDKGo=3L3oy?%J zGJJT`ci5BbE^aUPxtm^0+p0H))S)n{h+K0iROttp*bfF+K6v)+k4yru6Hp7=RmY;mgZPB>mGh) z)N4yGxp9K?jF=2&-N9dd?E|l4+4r>cs@@KYYSr$3o4j0f;di;HcNEijlDCI9AoF8_ zDk#p8Qb*X>Ws5mFJ@FrL(az5|hn%`ZiqVIrlL>h=#5MZQGU>M8tY^W+`z$l)lT3%B zfSd`33p5U@RTsy><*cohP67NWw*DpC~cRTRC zigqcv5{CSLoPv5+M)H$<3J)Zr7Hqd~^W; z{H{@wY zv7zDfz>3Y>3Fo8qS{oLAtVJ66;jQztZU56XRYs{v;+T3Mb$fxkYs)SGK;GRtnf=gC zE9sfCyUsHQg)>`>wJNKaYZ(ol0J{~S^lH3cBM)cZs%Ehp6R#9m@g!SAZF`T8U~G!X zx3rng(CO6X0_9ql@+>5t7Xi)hhJf{Z8Z+sZ8_RQ>|J~kIuesq9x9PBg&RJY^64auA zTYXCvUrYn#02~xn%^?q_J2&`bqn0P2U7csszX_XOKj-KI(B++gmeXjZwzb8lQ?uc| z4ONSjnSV%%(Z26*GwG5KWSLKMXYUw;<22C5`7Y@7PPB&9tIGf(oyM?qdFgkw6_eH57-SpB>r33(Zw-j@A|XgW7l`l`9IG!8o6!<1+d_=gH?)Jf`*7GthP$hG5&Z2> z-$iQ6V_u=;ipKMG@IO;)0}{P~g18PB=7HsmCyRzSglyG+<%jR%fIW>!aw>>-9yn)| zL9=|m2h~fNbFl7{irTk68wY;@7UhizF;8%KWScl&Rm;VGTv9olH5|a5fj-*8?W?^3 zSU;Z{m9cc{HW(}bR%>`Ci;0g012M3335>Sx%f!yR=55|o&IRhBq zhQac6^1{oftzpLT1KrPNR{0UVqiJ?Ivuc4;he#c*{T7e^HfaugxMUs8xQIZ@qBvo$ z$&xM9kS*5uNyZ}G*=X83fBWF->2Ttb!Kn-I{6Fqx;M5HN{Quq{S Date: Tue, 5 May 2026 16:05:01 +0600 Subject: [PATCH 5/6] fix changes Signed-off-by: urmi --- docs/guides/milvus/README.md | 72 --- docs/guides/milvus/_index.md | 10 - docs/guides/milvus/concepts/_index.md | 10 - docs/guides/milvus/concepts/catalog.md | 48 -- docs/guides/milvus/concepts/milvus.md | 548 ------------------ docs/guides/milvus/configuration/_index.md | 10 - .../milvus/configuration/using-config-file.md | 139 ----- docs/guides/milvus/custom-rbac/_index.md | 10 - .../milvus/custom-rbac/using-custom-rbac.md | 160 ----- docs/guides/milvus/monitoring/_index.md | 10 - docs/guides/milvus/monitoring/overview.md | 39 -- .../monitoring/using-builtin-prometheus.md | 149 ----- .../monitoring/using-prometheus-operator.md | 135 ----- docs/guides/milvus/ops-request/_index.md | 10 - docs/guides/milvus/ops-request/overview.md | 29 - docs/guides/milvus/private-registry/_index.md | 10 - .../using-private-registry.md | 130 ----- docs/guides/milvus/quickstart/_index.md | 10 - docs/guides/milvus/quickstart/quickstart.md | 158 ----- docs/guides/milvus/quickstart/rbac.md | 181 ------ docs/guides/milvus/tls/_index.md | 10 - docs/guides/milvus/tls/overview.md | 39 -- 22 files changed, 1917 deletions(-) delete mode 100644 docs/guides/milvus/README.md delete mode 100644 docs/guides/milvus/_index.md delete mode 100644 docs/guides/milvus/concepts/_index.md delete mode 100644 docs/guides/milvus/concepts/catalog.md delete mode 100644 docs/guides/milvus/concepts/milvus.md delete mode 100644 docs/guides/milvus/configuration/_index.md delete mode 100644 docs/guides/milvus/configuration/using-config-file.md delete mode 100644 docs/guides/milvus/custom-rbac/_index.md delete mode 100644 docs/guides/milvus/custom-rbac/using-custom-rbac.md delete mode 100644 docs/guides/milvus/monitoring/_index.md delete mode 100644 docs/guides/milvus/monitoring/overview.md delete mode 100644 docs/guides/milvus/monitoring/using-builtin-prometheus.md delete mode 100644 docs/guides/milvus/monitoring/using-prometheus-operator.md delete mode 100644 docs/guides/milvus/ops-request/_index.md delete mode 100644 docs/guides/milvus/ops-request/overview.md delete mode 100644 docs/guides/milvus/private-registry/_index.md delete mode 100644 docs/guides/milvus/private-registry/using-private-registry.md delete mode 100644 docs/guides/milvus/quickstart/_index.md delete mode 100644 docs/guides/milvus/quickstart/quickstart.md delete mode 100644 docs/guides/milvus/quickstart/rbac.md delete mode 100644 docs/guides/milvus/tls/_index.md delete mode 100644 docs/guides/milvus/tls/overview.md diff --git a/docs/guides/milvus/README.md b/docs/guides/milvus/README.md deleted file mode 100644 index 1a94c30eb..000000000 --- a/docs/guides/milvus/README.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: Milvus -menu: - docs_{{ .version }}: - identifier: milvus-readme - name: Milvus - parent: milvus-guides - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides -url: /docs/{{ .version }}/guides/milvus/ -aliases: - - /docs/{{ .version }}/guides/milvus/README/ ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Overview - -KubeDB supports vector database deployment with Milvus using the `Milvus` CRD. - -## Supported Milvus Features - -| Features | Availability | -|----------------------------------|:------------:| -| Standalone provisioning | ✓ | -| Distributed provisioning | ✓ | -| Monitoring | ✓ | -| TLS | No | -| Ops Requests | No | - -## Example Milvus Manifest - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: milvus-cluster - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: "my-release-minio" - topology: - mode: Distributed - distributed: - mixcoord: - replicas: 2 - storageType: Durable - storage: - accessModes: - - ReadWriteOnce - storageClassName: local-path - resources: - requests: - storage: 10Gi -``` - -## User Guide - -- [Quickstart Milvus](/docs/guides/milvus/quickstart/quickstart.md) with KubeDB operator. -- [Milvus CRD Concept](/docs/guides/milvus/concepts/milvus.md). -- [MilvusVersion CRD Concept](/docs/guides/milvus/concepts/catalog.md). -- [MilvusOpsRequest CRD Concept](/docs/guides/milvus/concepts/opsrequest.md). -- [RBAC Quickstart](/docs/guides/milvus/quickstart/rbac.md) -- [Private Registry](/docs/guides/milvus/private-registry/using-private-registry.md) -- [Custom RBAC](/docs/guides/milvus/custom-rbac/using-custom-rbac.md) -- [Custom Configuration](/docs/guides/milvus/configuration/using-config-file.md) -- [Monitoring](/docs/guides/milvus/monitoring/overview.md) for metrics collection guidance. -- [Builtin Prometheus Monitoring](/docs/guides/milvus/monitoring/using-builtin-prometheus.md) -- [Prometheus Operator Monitoring](/docs/guides/milvus/monitoring/using-prometheus-operator.md) \ No newline at end of file diff --git a/docs/guides/milvus/_index.md b/docs/guides/milvus/_index.md deleted file mode 100644 index 2b29e986f..000000000 --- a/docs/guides/milvus/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Milvus -menu: - docs_{{ .version }}: - identifier: milvus-guides - name: Milvus - parent: guides - weight: 15 -menu_name: docs_{{ .version }} ---- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/_index.md b/docs/guides/milvus/concepts/_index.md deleted file mode 100644 index f522969c1..000000000 --- a/docs/guides/milvus/concepts/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Milvus Concepts -menu: - docs_{{ .version }}: - identifier: milvus-concepts - name: Concepts - parent: milvus-guides - weight: 15 -menu_name: docs_{{ .version }} ---- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/catalog.md b/docs/guides/milvus/concepts/catalog.md deleted file mode 100644 index 077691bcc..000000000 --- a/docs/guides/milvus/concepts/catalog.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -title: MilvusVersion CRD -menu: - docs_{{ .version }}: - identifier: milvus-catalog-concepts - name: MilvusVersion - parent: milvus-concepts-milvus - weight: 15 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# MilvusVersion - -## What is MilvusVersion - -`MilvusVersion` is the catalog CRD that defines the Milvus engine image and related metadata for KubeDB-managed Milvus deployments. - -KubeDB uses this CRD when resolving `Milvus.spec.version`. - -## MilvusVersion Specification - -```yaml -apiVersion: catalog.kubedb.com/v1alpha1 -kind: MilvusVersion -metadata: - name: "2.6.11" -spec: - version: "2.6.11" - db: - image: "kubedb/milvus:2.6.11" - etcdVersion: v3.5.21 - deprecated: false -``` - -## Key fields - -- `metadata.name` is referenced from `Milvus.spec.version`. -- `spec.version` is the Milvus release version. -- `spec.db.image` is the image used for Milvus components. -- `spec.deprecated` indicates whether the entry is deprecated. - -## Next Steps - -- Read the [Milvus CRD concept](/docs/guides/milvus/concepts/milvus.md). -- Run the [Milvus quickstart](/docs/guides/milvus/quickstart/quickstart.md). \ No newline at end of file diff --git a/docs/guides/milvus/concepts/milvus.md b/docs/guides/milvus/concepts/milvus.md deleted file mode 100644 index 06bc2fe8a..000000000 --- a/docs/guides/milvus/concepts/milvus.md +++ /dev/null @@ -1,548 +0,0 @@ ---- -title: Milvus CRD -menu: - docs_{{ .version }}: - identifier: milvus-concepts-milvus - name: Milvus - parent: milvus-concepts - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Milvus - -## What is Milvus - -`Milvus` is a KubeDB `CustomResourceDefinition` used to deploy and manage Milvus vector databases. You only need to describe the desired database configuration in a `Milvus`object, and the KubeDB operator will create Kubernetes objects in the desired state for you. - -## Milvus Spec - -As with all other Kubernetes objects, a Milvus needs `apiVersion`, `kind`, and `metadata` fields. It also needs a `.spec` section. Below is an example of Milvus object. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: milvus-cluster - namespace: kubedb -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: "my-release-minio" - metaStorage: - externallyManaged: true - endpoints: - - http://etcdcluster-sample-0.etcdcluster-sample.default.svc.cluster.local:2379 - - http://etcdcluster-sample-1.etcdcluster-sample.default.svc.cluster.local:2379 - - http://etcdcluster-sample-2.etcdcluster-sample.default.svc.cluster.local:2379 - disableSecurity: false - authSecret: - name: "milvus-auth" - externallyManaged: true - configuration: - secretName: my-release-user-config - inline: - milvus.yaml: | - log: - level: info - file: - maxAge: 30 - topology: - mode: Distributed - distributed: - mixcoord: - replicas: 2 - podTemplate: - spec: - containers: - - name: milvus - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 600m - memory: 2Gi - - datanode: - replicas: 2 - podTemplate: - spec: - containers: - - name: milvus - resources: - requests: - cpu: 600m - memory: 1Gi - limits: - cpu: 700m - memory: 3Gi - - proxy: - replicas: 2 - podTemplate: - spec: - containers: - - name: milvus - resources: - requests: - cpu: 500m - memory: 2Gi - limits: - cpu: 600m - memory: 4Gi - querynode: - replicas: 2 - podTemplate: - spec: - containers: - - name: milvus - resources: - requests: - cpu: 800m - memory: 3Gi - limits: - cpu: 900m - memory: 4Gi - streamingnode: - replicas: 3 - podTemplate: - spec: - containers: - - name: milvus - resources: - requests: - cpu: 600m - memory: 2Gi - limits: - cpu: 700m - memory: 2Gi - storageType: Durable - storage: - accessModes: - - ReadWriteOnce - storageClassName: local-path - resources: - requests: - storage: 2Gi - monitor: - agent: prometheus.io/operator - prometheus: - exporter: - port: 9091 - resources: - limits: - memory: 512Mi - requests: - cpu: 600m - memory: 256Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - serviceMonitor: - interval: 10s - labels: - release: prometheus - tls: - issuerRef: - name: milvus-issuer - kind: Issuer - apiGroup: "cert-manager.io" - external: - mode: mTLS - internal: - mode: TLS - deletionPolicy: WipeOut - healthChecker: - periodSeconds: 15 - timeoutSeconds: 10 - failureThreshold: 2 - disableWriteCheck: false -``` - -### spec.version - -`spec.version` is a required field specifying the name of the [MilvusVersion](/docs/guides/milvus/concepts/milvusversion.md) crd where the docker images are specified. Currently, when you install KubeDB, it creates the following `Milvus` resources, - -- `2.6.7` -- `2.6.9` -- `2.6.11` - -### spec.objectStorage - -`spec.objectStorage` is a required field that specifies the object storage backend used by Milvus. Milvus depends on external object storage (such as MinIO or any S3-compatible service) to store its data. - -The configuration is provided via a Kubernetes `Secret` referenced in this field: - -```yaml -objectStorage: - configSecret: - name: my-release-minio -``` -The referenced secret must contain the following keys: - -- address – endpoint of the object storage service -- accesskey – username for authentication -- secretkey – password for authentication - -All values must be base64-encoded. - -In this setup, MinIO is deployed separately (for example, via Helm) and acts as a dependency for Milvus. KubeDB does not manage MinIO directly; it only uses the credentials provided through the secret to connect to the object storage. - -### spec.metaStorage - -`spec.metaStorage` defines how the metadata store (etcd) used by Milvus is configured. Milvus relies on etcd to manage cluster metadata and coordination. The etcd operator must be installed and running in the user cluster. - -- `externallyManaged` indicates whether the etcd cluster is managed outside of KubeDB. - - If true, users must provide the etcd endpoints. - - If false, KubeDB will create and manage an EtcdCluster resource. - -- `endpoints` is the list of etcd client endpoints and required when externallyManaged: true. -- `size (optional)` is the number of etcd nodes to provision and used only when externallyManaged: false. -- `storageType (optional)` defines storage behavior (e.g., durable or ephemeral). -- `storage (optional)` specifies the PersistentVolume configuration for etcd when managed by KubeDB. - -### spec.authSecret - -`spec.authSecret` is an optional field that points to a Secret used to hold credentials for `milvus` root user. If not set, KubeDB operator creates a new Secret `{milvus-object-name}-auth` for storing the password for `root` user for each Milvus object. - -We can use this field in 3 mode. -1. Using an external secret. In this case, You need to create an auth secret first with required fields, then specify the secret name when creating the Milvus object using `spec.authSecret.name` & set `spec.authSecret.externallyManaged` to true. -```yaml -authSecret: - name: - externallyManaged: true -``` - -2. Specifying the secret name only. In this case, You need to specify the secret name when creating the Milvus object using `spec.authSecret.name`. `externallyManaged` is by default false. -```yaml -authSecret: - name: -``` - -3. Let KubeDB do everything for you. In this case, no work for you. - -AuthSecret contains a `user` key and a `password` key which contains the `username` and `password` respectively for Milvus `root` user. - -Example: - -```bash -$ kubectl create secret generic milvus-auth -n demo \ ---from-literal=username=root \ ---from-literal=password=6q8u_2jMOW-OOZXk -secret "milvus-auth" created -``` - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: milvus-auth - namespace: kubedb -type: kubernetes.io/basic-auth -stringData: - username: "root" - password: "Milvus" -``` - -Secrets provided by users are not managed by KubeDB, and therefore, won't be modified or garbage collected by the KubeDB operator (version 0.13.0 and higher). - -### spec.configuration -`spec.configuration` is an optional field that specifies custom configuration for Milvus cluster. It has the following fields: -- `configuration.secretName` is an optional field that specifies the name of the secret that holds custom configuration files for Milvus cluster. -- `configuration.inline` is an optional field that allows you to provide custom configuration directly in the Milvus object. - -```yaml -configuration: - secretName: my-release-user-config - inline: - milvus.yaml: | - log: - level: info - file: - maxAge: 30 -``` - -### spec.topology - -`spec.topology` defines the deployment topology for Milvus. It supports both **Standalone** and **Distributed** (cluster) modes. - -### spec.topology.mode - -Specifies the deployment mode of Milvus. - -- **`Standalone`**: Runs Milvus as a single-node deployment. All components run inside a single pod. -- **`Distributed`**: Runs Milvus as a multi-component distributed cluster. - -### spec.topology.standalone - -```yaml -topology: - mode: Standalone -``` -When `mode: Standalone` is used: -- All Milvus components run in a single unified deployment. -- No separate component configuration (like datanode, proxy, etc.) is required. -- KubeDB manages all internal services automatically - -### spec.topology.distributed - -`distributed` contains the configuration for all Milvus components in distributed mode. - -#### spec.topology.mixcoord - -`mixcoord` is responsible for coordinating metadata and internal cluster orchestration. - -- **replicas**: Number of mixcoord pods (default: `1`) -- **podTemplate**: Custom resource requests/limits and pod-level configuration - - -#### spec.topology.distributed.datanode - -`datanode` handles data ingestion and persistence into storage. - -- **replicas**: Number of datanode pods (default: `1`) -- **podTemplate**: Resource configuration for each datanode pod - -#### spec.topology.distributed.proxy - -`proxy` is the entry point for client requests. - -- **replicas**: Number of proxy pods (default: `1`) -- **podTemplate**: Resource configuration for proxy pods - -#### spec.topology.distributed.querynode - -`querynode` executes search and query operations on vector data. - -- **replicas**: Number of querynode pods (default: `1`) -- **podTemplate**: Resource configuration for query execution workload - - -#### spec.topology.distributed.streamingnode - -`streamingnode` handles real-time streaming ingestion. - -- **replicas**: Number of streamingnode pods (default: `1`) -- **podTemplate**: Resource configuration for streaming workloads - -Additional storage configuration for `streamingnode`: - -- **storageType**: Defines storage behavior (`Durable` or `Ephemeral`) -- **storage**: PVC specification used when `storageType` is `Durable` - -### spec..podTemplate - -KubeDB allows providing a template for database pod through `spec..podTemplate`. KubeDB operator will pass the information provided in `spec..podTemplate` to the PetSet created for Milvus cluster. - -KubeDB accept following fields to set in `spec..podTemplate:` - -- metadata: - - annotations (pod's annotation) - - labels (pod's labels) -- controller: - - annotations (petset's annotation) - - labels (petset's labels) -- spec: - - containers - - volumes - - podPlacementPolicy - - initContainers - - containers - - imagePullSecrets - - nodeSelector - - serviceAccountName - - schedulerName - - tolerations - - priorityClassName - - priority - - securityContext - -You can check out the full list [here](https://github.com/kmodules/offshoot-api/blob/master/api/v2/types.go#L26C1-L279C1). -Uses of some field of `spec..podTemplate` is described below, - -#### spec..podTemplate.spec.tolerations - -The `spec.podTemplate.spec.tolerations` is an optional field. This can be used to specify the pod's tolerations. - -#### spec..podTemplate.spec.volumes - -The `spec..podTemplate..volumes` is an optional field. This can be used to provide the list of volumes that can be mounted by containers belonging to the pod. - -#### spec..podTemplate.spec.podPlacementPolicy - -`spec.podTemplate.spec.podPlacementPolicy` is an optional field. This can be used to provide the reference of the `podPlacementPolicy`. `name` of the podPlacementPolicy is referred under this attribute. This will be used by our Petset controller to place the db pods throughout the region, zone & nodes according to the policy. It utilizes kubernetes affinity & podTopologySpreadContraints feature to do so. -```yaml -spec: - podPlacementPolicy: - name: default -``` - -#### spec..podTemplate.spec.nodeSelector - -`spec..podTemplate.spec.nodeSelector` is an optional field that specifies a map of key-value pairs. For the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). To learn more, see [here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) . - -#### spec..podTemplate.spec.containers - -The `spec..podTemplate.spec.containers` can be used to provide the list containers and their configurations for to the database pod. some of the fields are described below, - -##### spec..podTemplate.spec.containers[].name -The `spec..podTemplate.spec.containers[].name` field used to specify the name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. - -##### spec..podTemplate.spec.containers[].args -`spec..podTemplate.spec.containers[].args` is an optional field. This can be used to provide additional arguments to database installation. - -##### spec..podTemplate.spec.containers[].env - -`spec..podTemplate.spec.containers[].env` is an optional field that specifies the environment variables to pass to the Redis containers. - -##### spec..podTemplate.spec.containers[].resources - -`spec..podTemplate.spec.containers[].resources` is an optional field. This can be used to request compute resources required by containers of the database pods. To learn more, visit [here](http://kubernetes.io/docs/user-guide/compute-resources/). - -### spec.serviceTemplates - -You can also provide template for the services created by KubeDB operator for Milvus cluster through `spec.serviceTemplates`. This will allow you to set the type and other properties of the services. - -KubeDB allows following fields to set in `spec.serviceTemplates`: -- `alias` represents the identifier of the service. It has the following possible value: - - `stats` for is used for the `exporter` service identification. - -Milvus comes with one primary services used for client access (Standalone or Proxy in Distributed mode) and four component services for distributed mode (`mixcoord`, `datanode`, `querynode` and `streamingnode`). There are two options for providing serviceTemplates: -- To provide `serviceTemplates` for a specific service, the `serviceTemplates.ports.port` should be equal to the port of that service and `serviceTemplate` will be used for that particular service only. -- However, to provide a common `serviceTemplates`, `serviceTemplates.ports.port` should be empty. - -- metadata: - - labels - - annotations -- spec: - - type - - ports - - clusterIP - - externalIPs - - loadBalancerIP - - loadBalancerSourceRanges - - externalTrafficPolicy - - healthCheckNodePort - - sessionAffinityConfig - -See [here](https://github.com/kmodules/offshoot-api/blob/kubernetes-1.21.1/api/v1/types.go#L237) to understand these fields in detail. - -### spec.monitor - -Milvus managed by KubeDB can be monitored with Prometheus operator out-of-the-box. To learn more, -- [Monitor Apache Milvus with Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md) - -### spec.tls - -`spec.tls` defines the TLS configuration for securing Milvus communication. The KubeDB operator uses [cert-manager](https://cert-manager.io/) to issue and manage certificates. Currently, only **PKCS#8 encoded certificates** are supported. - -TLS in Milvus can be configured for: -- **External traffic** (client → Milvus) -- **Internal traffic** (inter-component communication) - ---- - -### Example - -```yaml -spec: - tls: - issuerRef: - apiGroup: cert-manager.io - kind: Issuer - name: milvus-issuer - external: - mode: mTLS - internal: - mode: TLS -``` -The `spec.tls` contains the following fields: - -- `tls.issuerRef` - is an `optional` field that references to the `Issuer` or `ClusterIssuer` custom resource object of [cert-manager](https://cert-manager.io/docs/concepts/issuer/). It is used to generate the necessary certificate secrets for Milvus. If the `issuerRef` is not specified, the operator creates a self-signed CA and also creates necessary certificate (valid: 365 days) secrets using that CA. - - `apiGroup` - is the group name of the resource that is being referenced. Currently, the only supported value is `cert-manager.io`. - - `kind` - is the type of resource that is being referenced. The supported values are `Issuer` and `ClusterIssuer`. - - `name` - is the name of the resource ( `Issuer` or `ClusterIssuer` ) that is being referenced. - - -- `tls.external` - `external` controls TLS for client-facing traffic (gRPC / REST). - - `TLS` - requires only the server certificate to encrypt communication between client and Milvus proxy. - - `mTLS` - requires both server and client certificates to enable mutual authentication between client and Milvus proxy. - - -- `tls.internal` - `internal` enables TLS for inter-component communication within the Milvus cluster. - - `TLS` - uses only server-side certificates to encrypt communication between internal Milvus components (e.g., proxy, querynode, datanode). - - `mTLS` - not supported for internal communication; internal traffic is generally secured using one-way TLS only. - - -- `tls.certificates` - is an `optional` field that specifies a list of certificate configurations used to configure the certificates. It has the following fields: - - `alias` - represents the identifier of the certificate. It has the following possible value: - - `server` - is used for the server certificate configuration. - - `client` - is used for the client certificate configuration. - - - - `secretName` - ( `string` | `"-alias-cert"` ) - specifies the k8s secret name that holds the certificates. - - - - `subject` - specifies an `X.509` distinguished name (DN). It has the following configurable fields: - - `organizations` ( `[]string` | `nil` ) - is a list of organization names. - - `organizationalUnits` ( `[]string` | `nil` ) - is a list of organization unit names. - - `countries` ( `[]string` | `nil` ) - is a list of country names (ie. Country Codes). - - `localities` ( `[]string` | `nil` ) - is a list of locality names. - - `provinces` ( `[]string` | `nil` ) - is a list of province names. - - `streetAddresses` ( `[]string` | `nil` ) - is a list of street addresses. - - `postalCodes` ( `[]string` | `nil` ) - is a list of postal codes. - - `serialNumber` ( `string` | `""` ) is a serial number. - - For more details, visit [here](https://golang.org/pkg/crypto/x509/pkix/#Name). - - - - `duration` ( `string` | `""` ) - is the period during which the certificate is valid. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300m"`, `"1.5h"` or `"20h45m"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". - - `renewBefore` ( `string` | `""` ) - is a specifiable time before expiration duration. - - `dnsNames` ( `[]string` | `nil` ) - is a list of subject alt names. - - `ipAddresses` ( `[]string` | `nil` ) - is a list of IP addresses. - - `uris` ( `[]string` | `nil` ) - is a list of URI Subject Alternative Names. - - `emailAddresses` ( `[]string` | `nil` ) - is a list of email Subject Alternative Names. - -### spec.deletionPolicy - -`deletionPolicy` gives flexibility whether to `nullify`(reject) the delete operation of `Milvus` crd or which resources KubeDB should keep or delete when you delete `Milvus` crd. KubeDB provides following four deletion policies: - -- DoNotTerminate -- WipeOut -- Halt -- Delete - -When `deletionPolicy` is `DoNotTerminate`, KubeDB takes advantage of `ValidationWebhook` feature in Kubernetes 1.9.0 or later clusters to implement `DoNotTerminate` feature. If admission webhook is enabled, `DoNotTerminate` prevents users from deleting the database as long as the `spec.deletionPolicy` is set to `DoNotTerminate`. - -> For more details you can visit [here](https://appscode.com/blog/post/deletion-policy/) - -### spec.healthChecker -It defines the attributes for the health checker. -- `spec.healthChecker.periodSeconds` specifies how often to perform the health check. -- `spec.healthChecker.timeoutSeconds` specifies the number of seconds after which the probe times out. -- `spec.healthChecker.failureThreshold` specifies minimum consecutive failures for the healthChecker to be considered failed. -- `spec.healthChecker.disableWriteCheck` specifies whether to disable the writeCheck or not. - -Know details about KubeDB Health checking from this [blog post](https://appscode.com/blog/post/kubedb-health-checker/). - -## Next Steps - -- Learn how to use KubeDB to run Apache Milvus cluster [here](/docs/guides/milvus/README.md). -- Deploy [dedicated topology cluster](/docs/guides/milvus/clustering/guide/index.md) for Apache Milvus -- Monitor your Milvus cluster with KubeDB using [`out-of-the-box` Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md). -- Detail concepts of [MilvusVersion object](/docs/guides/milvus/concepts/milvusversion.md). - -[//]: # (- Learn to use KubeDB managed Milvus objects using [CLIs](/docs/guides/milvus/cli/cli.md).) -- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). \ No newline at end of file diff --git a/docs/guides/milvus/configuration/_index.md b/docs/guides/milvus/configuration/_index.md deleted file mode 100644 index 0224bf33d..000000000 --- a/docs/guides/milvus/configuration/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Run Milvus with Custom Configuration -menu: - docs_{{ .version }}: - identifier: milvus-configuration - name: Custom Configuration - parent: milvus-guides - weight: 130 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/configuration/using-config-file.md b/docs/guides/milvus/configuration/using-config-file.md deleted file mode 100644 index ac4888141..000000000 --- a/docs/guides/milvus/configuration/using-config-file.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: Run Milvus with Custom Configuration -menu: - docs_{{ .version }}: - identifier: milvus-using-config-file - name: Config File - parent: milvus-configuration - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Using Custom Configuration File - -KubeDB supports providing custom configuration for Milvus. This tutorial will show you how to use KubeDB to run Milvus with custom configuration. - -## Before You Begin - -- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -- Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). - -- To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. - -```bash -$ kubectl create ns demo -namespace/demo created -``` - -> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). - -## Overview - -Milvus supports configuration via the `milvus.yaml` file. KubeDB takes advantage of `spec.configuration.secretName` to allow users to provide a custom `milvus.yaml` without mounting any volume into the Pod. The operator reads this Secret internally and applies the configuration automatically. - -In this tutorial, we will configure `queryNode.gracefulTime` and `dataNode.segment.maxSize` parameters. - -## Custom Configuration - -At first, let's create a custom `milvus.yaml` file: - -```yaml -queryNode: - gracefulTime: 5000 - -dataNode: - segment: - maxSize: 512 -``` - -Now, create a Secret with this configuration file. - -```bash -$ kubectl create secret generic -n demo milvus-configuration \ - --from-file=milvus.yaml=./milvus.yaml -secret/milvus-configuration created -``` - -Verify the Secret has the configuration file. - -```bash -$ kubectl get secret -n demo milvus-configuration -o yaml -apiVersion: v1 -data: - milvus.yaml: -kind: Secret -metadata: - name: milvus-configuration - namespace: demo -``` - -Now, create Milvus CRD specifying `spec.configuration.secretName` field. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: custom-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - configuration: - secretName: milvus-configuration - storageType: Durable - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - deletionPolicy: WipeOut -``` - -```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/configuration/milvus-configuration.yaml -milvus.kubedb.com/custom-milvus created -``` - -Now, wait for the Milvus to be ready. - -```bash -$ kubectl get milvus -n demo custom-milvus -NAME VERSION STATUS AGE -custom-milvus 2.4.0 Ready 3m -``` - -Check that the pod is running: - -```bash -$ kubectl get pod -n demo custom-milvus-0 -NAME READY STATUS RESTARTS AGE -custom-milvus-0 1/1 Running 0 3m -``` - -Now, we will verify the configuration has been applied. We will `exec` into the pod and check the configuration file. - -```bash -$ kubectl exec -it -n demo custom-milvus-0 -- cat /milvus/configs/milvus.yaml | grep -A 5 queryNode -queryNode: - gracefulTime: 5000 -``` - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/custom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/custom-milvus - -kubectl delete -n demo secret milvus-configuration -kubectl delete ns demo -``` diff --git a/docs/guides/milvus/custom-rbac/_index.md b/docs/guides/milvus/custom-rbac/_index.md deleted file mode 100644 index d322e1d6d..000000000 --- a/docs/guides/milvus/custom-rbac/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Run Milvus with Custom RBAC resources -menu: - docs_{{ .version }}: - identifier: milvus-custom-rbac - name: Custom RBAC - parent: milvus-guides - weight: 90 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/custom-rbac/using-custom-rbac.md b/docs/guides/milvus/custom-rbac/using-custom-rbac.md deleted file mode 100644 index 718628af9..000000000 --- a/docs/guides/milvus/custom-rbac/using-custom-rbac.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: Run Milvus with Custom RBAC resources -menu: - docs_{{ .version }}: - identifier: milvus-custom-rbac-quickstart - name: Custom RBAC - parent: milvus-custom-rbac - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Using Custom RBAC Resources - -KubeDB supports finer user control over role based access permissions provided to a Milvus instance. This tutorial will show you how to use KubeDB to run Milvus instance with custom RBAC resources. - -## Before You Begin - -At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). - -To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. - -```bash -$ kubectl create ns demo -namespace/demo created -``` - -## Overview - -KubeDB allows users to provide custom RBAC resources, namely, `ServiceAccount`, `Role`, and `RoleBinding` for Milvus. This is provided via the `spec.podTemplate.spec.serviceAccountName` field in Milvus CRD. - -## Custom RBAC for Milvus - -At first, let's create a `Service Account` in `demo` namespace. - -```bash -$ kubectl create serviceaccount -n demo my-custom-serviceaccount -serviceaccount/my-custom-serviceaccount created -``` - -Now, we need to create a role that has necessary access permissions for the Milvus database named `quick-milvus`. - -```yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: my-custom-role - namespace: demo -rules: -- apiGroups: - - apps - resourceNames: - - quick-milvus - resources: - - petsets - verbs: - - get -- apiGroups: - - kubedb.com - resourceNames: - - quick-milvus - resources: - - milvuses - verbs: - - get -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - patch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - update -``` - -```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-role.yaml -role.rbac.authorization.k8s.io/my-custom-role created -``` - -Now create a `RoleBinding` to bind this `Role` with the already created service account. - -```bash -$ kubectl create rolebinding my-custom-rolebinding \ - --role=my-custom-role \ - --serviceaccount=demo:my-custom-serviceaccount \ - --namespace=demo -rolebinding.rbac.authorization.k8s.io/my-custom-rolebinding created -``` - -Now, create a Milvus CRD specifying `spec.podTemplate.spec.serviceAccountName` field to `my-custom-serviceaccount`. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: quick-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - storageType: Durable - podTemplate: - spec: - serviceAccountName: my-custom-serviceaccount - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - deletionPolicy: WipeOut -``` - -```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-db.yaml -milvus.kubedb.com/quick-milvus created -``` - -Check that the pod is running: - -```bash -$ kubectl get pod -n demo quick-milvus-0 -NAME READY STATUS RESTARTS AGE -quick-milvus-0 1/1 Running 0 3m -``` - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/quick-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/quick-milvus - -kubectl delete -n demo serviceaccount my-custom-serviceaccount -kubectl delete -n demo role my-custom-role -kubectl delete -n demo rolebinding my-custom-rolebinding -kubectl delete ns demo -``` diff --git a/docs/guides/milvus/monitoring/_index.md b/docs/guides/milvus/monitoring/_index.md deleted file mode 100644 index a1c83edfb..000000000 --- a/docs/guides/milvus/monitoring/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Milvus Monitoring -menu: - docs_{{ .version }}: - identifier: milvus-monitoring - name: Monitoring - parent: milvus-guides - weight: 20 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/monitoring/overview.md b/docs/guides/milvus/monitoring/overview.md deleted file mode 100644 index 248796e32..000000000 --- a/docs/guides/milvus/monitoring/overview.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Milvus Monitoring Overview -menu: - docs_{{ .version }}: - identifier: milvus-monitoring-overview - name: Overview - parent: milvus-monitoring - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -# Milvus Monitoring - -This guide shows how to enable and verify monitoring for Milvus deployments. - -## Before You Begin - -- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). -- Install Prometheus Operator or another compatible metrics collection stack. - -## Enable Monitoring - -Milvus supports monitoring integration through `spec.monitor`. - -- Configure monitoring agent and service monitor selectors. -- Validate scrape targets for proxy and distributed components. - -## Verify - -```bash -kubectl get milvus -n demo milvus-cluster -o yaml -kubectl get servicemonitor -A -``` - -## Next Steps - -- Review which Milvus components need dedicated scrape targets in distributed mode. -- Add dashboards for latency, indexing throughput, and node health. diff --git a/docs/guides/milvus/monitoring/using-builtin-prometheus.md b/docs/guides/milvus/monitoring/using-builtin-prometheus.md deleted file mode 100644 index 1b405cf87..000000000 --- a/docs/guides/milvus/monitoring/using-builtin-prometheus.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -title: Monitor Milvus using Builtin Prometheus Discovery -menu: - docs_{{ .version }}: - identifier: milvus-using-builtin-prometheus-monitoring - name: Builtin Prometheus - parent: milvus-monitoring - weight: 20 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Monitoring Milvus with Builtin Prometheus - -This tutorial will show you how to monitor Milvus database using builtin [Prometheus](https://github.com/prometheus/prometheus) scraper. - -## Before You Begin - -- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). - -- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/milvus/monitoring/overview.md). - -- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy the database in `demo` namespace. - - ```bash - $ kubectl create ns monitoring - namespace/monitoring created - - $ kubectl create ns demo - namespace/demo created - ``` - -> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). - -## Deploy Milvus with Monitoring Enabled - -At first, let's deploy a Milvus database with monitoring enabled. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: builtin-prom-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - storageType: Durable - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - deletionPolicy: WipeOut - monitor: - agent: prometheus.io/builtin -``` - -Here, `spec.monitor.agent: prometheus.io/builtin` specifies that we are going to monitor this server using builtin Prometheus scraper. - -```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/monitoring/builtin-prom-milvus.yaml -milvus.kubedb.com/builtin-prom-milvus created -``` - -Now, wait for the database to go into `Ready` state. - -```bash -$ kubectl get milvus -n demo builtin-prom-milvus -NAME VERSION STATUS AGE -builtin-prom-milvus 2.4.0 Ready 2m -``` - -KubeDB will create a separate stats service with name `{Milvus crd name}-stats` for monitoring purpose. - -```bash -$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=builtin-prom-milvus" -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -builtin-prom-milvus ClusterIP 10.96.100.20 19530/TCP 2m -builtin-prom-milvus-stats ClusterIP 10.96.100.21 9091/TCP 90s -``` - -Let's describe the stats service: - -```bash -$ kubectl describe svc -n demo builtin-prom-milvus-stats -Name: builtin-prom-milvus-stats -Namespace: demo -Labels: app.kubernetes.io/name=milvuses.kubedb.com - app.kubernetes.io/instance=builtin-prom-milvus -Annotations: monitoring.appscode.com/agent: prometheus.io/builtin - prometheus.io/path: /metrics - prometheus.io/port: 9091 - prometheus.io/scrape: true -``` - -The service contains the following annotations which are used by builtin Prometheus to discover the endpoint: - -``` -prometheus.io/path: /metrics -prometheus.io/port: 9091 -prometheus.io/scrape: true -``` - -Configure your Prometheus to scrape metrics from the `monitoring` namespace service discovery: - -```yaml -scrape_configs: -- job_name: kubedb-milvuses - honor_labels: true - kubernetes_sd_configs: - - role: endpoints - namespaces: - names: - - demo - relabel_configs: - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] - action: keep - regex: true - - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] - action: replace - target_label: __metrics_path__ - regex: (.+) - - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] - action: replace - target_label: __address__ - regex: ([^:]+)(?::\d+)?;(\d+) - replacement: $1:$2 -``` - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/builtin-prom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/builtin-prom-milvus - -kubectl delete ns demo -kubectl delete ns monitoring -``` diff --git a/docs/guides/milvus/monitoring/using-prometheus-operator.md b/docs/guides/milvus/monitoring/using-prometheus-operator.md deleted file mode 100644 index 7eac16d3b..000000000 --- a/docs/guides/milvus/monitoring/using-prometheus-operator.md +++ /dev/null @@ -1,135 +0,0 @@ ---- -title: Monitor Milvus using Prometheus Operator -menu: - docs_{{ .version }}: - identifier: milvus-using-prometheus-operator-monitoring - name: Prometheus Operator - parent: milvus-monitoring - weight: 30 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Monitoring Milvus using Prometheus Operator - -[Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) provides simple and Kubernetes native way to deploy and configure Prometheus, Alertmanager and related monitoring components. This tutorial will show you how to use the Prometheus operator to monitor Milvus database deployed with KubeDB. - -## Before You Begin - -- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). - -- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/milvus/monitoring/overview.md). - -- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy the database in `demo` namespace. - - ```bash - $ kubectl create ns monitoring - namespace/monitoring created - - $ kubectl create ns demo - namespace/demo created - ``` - -> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). - -## Deploy Milvus with Monitoring Enabled - -At first, let's deploy a Milvus database with monitoring enabled. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: coreos-prom-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - storageType: Durable - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - deletionPolicy: WipeOut - monitor: - agent: prometheus.io/operator - prometheus: - serviceMonitor: - labels: - release: prometheus - interval: 10s -``` - -Here, -- `spec.monitor.agent: prometheus.io/operator` indicates that we are going to monitor this server using Prometheus operator. -- `spec.monitor.prometheus.serviceMonitor.labels` specifies that KubeDB should create `ServiceMonitor` with these labels. -- `spec.monitor.prometheus.serviceMonitor.interval` indicates that the Prometheus should scrape metrics from this database with 10 seconds interval. - -```bash -$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/monitoring/coreos-prom-milvus.yaml -milvus.kubedb.com/coreos-prom-milvus created -``` - -Now, wait for the database to go into `Ready` state. - -```bash -$ kubectl get milvus -n demo coreos-prom-milvus -NAME VERSION STATUS AGE -coreos-prom-milvus 2.4.0 Ready 2m -``` - -KubeDB will create a `ServiceMonitor` CRD in the same namespace as the Milvus database. - -```bash -$ kubectl get servicemonitor -n demo -NAME AGE -coreos-prom-milvus 2m -``` - -Let's verify the `ServiceMonitor` has the right label to be discovered by Prometheus: - -```bash -$ kubectl get servicemonitor -n demo coreos-prom-milvus -o yaml -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - labels: - release: prometheus - name: coreos-prom-milvus - namespace: demo -spec: - endpoints: - - honorLabels: true - interval: 10s - path: /metrics - port: metrics - namespaceSelector: - matchNames: - - demo - selector: - matchLabels: - app.kubernetes.io/instance: coreos-prom-milvus -``` - -Now, if we go to the Prometheus dashboard, we should see the target being scraped. - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/coreos-prom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/coreos-prom-milvus - -kubectl delete ns demo -kubectl delete ns monitoring -``` diff --git a/docs/guides/milvus/ops-request/_index.md b/docs/guides/milvus/ops-request/_index.md deleted file mode 100644 index 9d0b5d697..000000000 --- a/docs/guides/milvus/ops-request/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Ops Request -menu: - docs_{{ .version }}: - identifier: milvus-ops-request - name: Ops Request - parent: milvus-guides - weight: 30 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/ops-request/overview.md b/docs/guides/milvus/ops-request/overview.md deleted file mode 100644 index ae0b7e728..000000000 --- a/docs/guides/milvus/ops-request/overview.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Milvus Ops Request Overview -menu: - docs_{{ .version }}: - identifier: milvus-ops-request-overview - name: Overview - parent: milvus-ops-request - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -# Milvus Ops Request - -This page tracks Milvus operations supported by the new database matrix used for these guides. - -## Before You Begin - -- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). -- Confirm the current support list in [new_db.md](/new_db.md) for your release. - -## Current Status - -Based on [new_db.md](/new_db.md), no Milvus ops request types are listed yet. - -## What to Check Next - -- Watch future KubeDB releases for Milvus `OpsRequest` support. -- Use the TLS and monitoring guides for common day-2 platform tasks. diff --git a/docs/guides/milvus/private-registry/_index.md b/docs/guides/milvus/private-registry/_index.md deleted file mode 100644 index c202e6450..000000000 --- a/docs/guides/milvus/private-registry/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Run Milvus using Private Registry -menu: - docs_{{ .version }}: - identifier: milvus-private-registry - name: Private Registry - parent: milvus-guides - weight: 120 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/private-registry/using-private-registry.md b/docs/guides/milvus/private-registry/using-private-registry.md deleted file mode 100644 index ab2852652..000000000 --- a/docs/guides/milvus/private-registry/using-private-registry.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -title: Run Milvus using Private Registry -menu: - docs_{{ .version }}: - identifier: milvus-using-private-registry - name: Quickstart - parent: milvus-private-registry - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Using Private Docker Registry - -KubeDB supports using private Docker registries. This tutorial will show you how to run KubeDB managed Milvus database using private Docker images. - -## Before You Begin - -At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. - -```bash -$ kubectl create ns demo -namespace/demo created -``` - -## Prepare Private Docker Registry - -- You will also need a docker private [registry](https://docs.docker.com/registry/) or [private repository](https://docs.docker.com/docker-hub/repos/#private-repositories). - -- Push the required images from KubeDB's [Docker hub account](https://hub.docker.com/r/kubedb/) into your private registry. For Milvus, push the `DB_IMAGE` of the following MilvusVersions, where `deprecated` is not true. - - ```bash - $ kubectl get milvusversions -o=custom-columns=NAME:.metadata.name,VERSION:.spec.version,DB_IMAGE:.spec.db.image,DEPRECATED:.spec.deprecated - NAME VERSION DB_IMAGE DEPRECATED - 2.4.0 2.4.0 kubedb/milvus:2.4.0 - ``` - -## Create ImagePullSecret - -Run the following command to create an image pull secret for your private Docker registry: - -```bash -$ kubectl create secret docker-registry -n demo myregistrykey \ - --docker-server=DOCKER_REGISTRY_SERVER \ - --docker-username=DOCKER_USER \ - --docker-email=DOCKER_EMAIL \ - --docker-password=DOCKER_PASSWORD -secret/myregistrykey created -``` - -## Install KubeDB Operator - -Follow the steps to [install KubeDB operator](/docs/setup/README.md) properly in cluster so that it points to the DOCKER_REGISTRY you wish to pull images from. - -## Create MilvusVersion CRD - -Create a MilvusVersion CRD specifying images from your private registry. Replace `PRIVATE_REGISTRY` with your private registry. - -```yaml -apiVersion: catalog.kubedb.com/v1alpha1 -kind: MilvusVersion -metadata: - name: "2.6.11" -spec: - db: - image: PRIVATE_REGISTRY/milvus:2.6.11 - etcdVersion: v3.5.21 - version: "2.6.11" -``` - -```bash -$ kubectl apply -f pvt-milvusversion.yaml -milvusversion.catalog.kubedb.com/2.4.0 created -``` - -## Deploy Milvus from Private Registry - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: pvt-reg-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - storageType: Durable - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - podTemplate: - spec: - imagePullSecrets: - - name: myregistrykey - deletionPolicy: WipeOut -``` - -```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/private-registry/pvt-reg-milvus.yaml -milvus.kubedb.com/pvt-reg-milvus created -``` - -Check that the Milvus is in Running state: - -```bash -$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=pvt-reg-milvus" -NAME READY STATUS RESTARTS AGE -pvt-reg-milvus-0 1/1 Running 0 3m -``` - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/pvt-reg-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/pvt-reg-milvus - -kubectl delete ns demo -``` diff --git a/docs/guides/milvus/quickstart/_index.md b/docs/guides/milvus/quickstart/_index.md deleted file mode 100644 index 753aefff9..000000000 --- a/docs/guides/milvus/quickstart/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Milvus Quickstart -menu: - docs_{{ .version }}: - identifier: milvus-quickstart - name: Quickstart - parent: milvus-guides - weight: 15 -menu_name: docs_{{ .version }} ---- \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/quickstart.md b/docs/guides/milvus/quickstart/quickstart.md deleted file mode 100644 index 9078abdab..000000000 --- a/docs/guides/milvus/quickstart/quickstart.md +++ /dev/null @@ -1,158 +0,0 @@ ---- -title: Milvus Quickstart -menu: - docs_{{ .version }}: - identifier: milvus-quickstart-overview - name: Overview - parent: milvus-quickstart - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# Milvus QuickStart - -This tutorial shows how to run a Milvus database with KubeDB. - -

-  lifecycle -

- -> Note: YAML files used in this tutorial are stored in [docs/examples/milvus/quickstart](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus/quickstart). - -## Before You Begin - -At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md) and make sure to include the flags `--set global.featureGates.Milvus=true` to ensure **Milvus CRD**. - -To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. - -```bash -$ kubectl create namespace demo -namespace/demo created - -$ kubectl get namespace -NAME STATUS AGE -demo Active 9s -``` - -## Check Available MilvusVersion - -When you install the KubeDB operator, it registers a CRD named [MilvusVersion](/docs/guides/milvus/concepts/catalog.md). The installation process comes with a set of tested MilvusVersion objects. Let's check available MilvusVersions by, - -```bash -$ kubectl get milvusversions -NAME VERSION DB_IMAGE DEPRECATED AGE -2.6.11 2.6.11 ghcr.io/appscode-images/milvus:2.6.11 6d2h -2.6.7 2.6.7 ghcr.io/appscode-images/milvus:2.6.7 6d2h -2.6.9 2.6.9 ghcr.io/appscode-images/milvus:2.6.9 6d2h -``` - -Notice the `DEPRECATED` column. Here, `true` means that this MilvusVersion is deprecated for the current KubeDB version. KubeDB will not work for deprecated MilvusVersion. You can also use the short from `mvversion` to check available MilvusVersions. - -In this tutorial, we will use `2.6.11` MilvusVersion CR to create a Milvus cluster. - -## Get External Dependencies Ready - -### Object Storage - -One of the external dependency of Milvus is object storage where the segments are stored. It is a storage mechanism that Milvus does not provide. **S3-compatible storage** (like **Minio**) are generally convenient options for object storage. - -In this tutorial, we will run a `minio-server` as object storage in our local `kind` cluster using `minio-operator` and create a bucket named `milvus` in it, which the deployed milvus database will use. - -```bash - -$ helm repo add minio https://operator.min.io/ -$ helm repo update minio -$ helm upgrade --install --namespace "minio-operator" --create-namespace "minio-operator" minio/operator --set operator.replicaCount=1 - -$ helm upgrade --install --namespace "demo" --create-namespace milvus-minio minio/tenant \ ---set tenant.pools[0].servers=1 \ ---set tenant.pools[0].volumesPerServer=1 \ ---set tenant.pools[0].size=1Gi \ ---set tenant.certificate.requestAutoCert=false \ ---set tenant.buckets[0].name="milvus" \ ---set tenant.pools[0].name="default" - -``` - -Now we need to create a `Secret` named `my-release-minio`. It contains the necessary connection information using which the milvus database will connect to the object storage. - -```yaml -apiVersion: v1 -kind: Secret -metadata: - name: my-release-minio - namespace: demo -stringData: - milvus.storage.type: "s3" - milvus.storage.bucket: "milvus" - milvus.storage.baseKey: "milvus/segments" - milvus.s3.accessKey: "minio" - milvus.s3.secretKey: "minio123" - milvus.s3.protocol: "http" - milvus.s3.enablePathStyleAccess: "true" - milvus.s3.endpoint.signingRegion: "us-east-1" - milvus.s3.endpoint.url: "http://myminio-hl.demo.svc.cluster.local:9000/" -``` - -Let’s create the `deep-storage-config` Secret shown above: - -```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/deep-storage-config.yaml -secret/deep-storage-config created -``` - -You can also use options like **Amazon S3**, **Google Cloud Storage**, **Azure Blob Storage** or **HDFS** and create a connection information `Secret` like this, and you are good to go. - -## Create a Milvus Database - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: milvus-cluster - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: "my-release-minio" - topology: - mode: Distributed - distributed: - mixcoord: - replicas: 2 - storageType: Durable - storage: - accessModes: - - ReadWriteOnce - storageClassName: local-path - resources: - requests: - storage: 10Gi -``` - -```bash -kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/distributed.yaml -kubectl get milvus -n demo milvus-cluster -w -``` - -## Verify Milvus Database - -```bash -kubectl get milvus -n demo -kubectl describe milvus -n demo milvus-cluster -``` - -When `status.phase` becomes `Ready`, the Milvus deployment is ready to serve vector search traffic. - -## Cleaning up - -```bash -kubectl delete milvus -n demo milvus-cluster -kubectl delete ns demo -``` \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/rbac.md b/docs/guides/milvus/quickstart/rbac.md deleted file mode 100644 index b4f50577e..000000000 --- a/docs/guides/milvus/quickstart/rbac.md +++ /dev/null @@ -1,181 +0,0 @@ ---- -title: RBAC for Milvus -menu: - docs_{{ .version }}: - identifier: milvus-rbac-quickstart - name: RBAC - parent: milvus-quickstart - weight: 15 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -> New to KubeDB? Please start [here](/docs/README.md). - -# RBAC Permissions for Milvus - -When RBAC is enabled in your cluster, KubeDB automatically creates the necessary Role, ServiceAccount, and RoleBinding for each Milvus instance. This tutorial explains what permissions are granted and how to verify them. - -Here is the list of additional permissions required by the PetSet of Milvus: - -| Kubernetes Resource | Resource Names | Permission required | -|---------------------|-------------------|---------------------| -| petsets | `{milvus-name}` | get | -| pods | | list, patch | -| pods/exec | | create | -| milvuses | | get | -| configmaps | `{milvus-name}` | get, update, create | - -## Before You Begin - -- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). - -- Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). - -- To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. - -```bash -$ kubectl create ns demo -namespace/demo created -``` - -## Create a Milvus Database - -Below is the Milvus object created in this tutorial. - -```yaml -apiVersion: kubedb.com/v1alpha2 -kind: Milvus -metadata: - name: quick-milvus - namespace: demo -spec: - version: "2.6.11" - objectStorage: - configSecret: - name: my-release-minio - storageType: Durable - storage: - storageClassName: "standard" - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - deletionPolicy: Delete -``` - -Create the above Milvus object: - -```bash -$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/quick-milvus.yaml -milvus.kubedb.com/quick-milvus created -``` - -When this Milvus object is created, KubeDB operator creates Role, ServiceAccount and RoleBinding with the matching Milvus name. - -### Role - -```bash -$ kubectl get role -n demo quick-milvus -o yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - app.kubernetes.io/component: database - app.kubernetes.io/instance: quick-milvus - app.kubernetes.io/managed-by: kubedb.com - app.kubernetes.io/name: milvuses.kubedb.com - name: quick-milvus - namespace: demo -rules: -- apiGroups: - - apps - resourceNames: - - quick-milvus - resources: - - petsets - verbs: - - get -- apiGroups: - - kubedb.com - resourceNames: - - quick-milvus - resources: - - milvuses - verbs: - - get -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - patch -- apiGroups: - - "" - resources: - - pods/exec - verbs: - - create -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - update -``` - -### ServiceAccount - -```bash -$ kubectl get serviceaccount -n demo quick-milvus -o yaml -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app.kubernetes.io/component: database - app.kubernetes.io/instance: quick-milvus - app.kubernetes.io/managed-by: kubedb.com - app.kubernetes.io/name: milvuses.kubedb.com - name: quick-milvus - namespace: demo -``` - -### RoleBinding - -```bash -$ kubectl get rolebinding -n demo quick-milvus -o yaml -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - app.kubernetes.io/component: database - app.kubernetes.io/instance: quick-milvus - app.kubernetes.io/managed-by: kubedb.com - app.kubernetes.io/name: milvuses.kubedb.com - name: quick-milvus - namespace: demo -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: quick-milvus -subjects: -- kind: ServiceAccount - name: quick-milvus - namespace: demo -``` - -## Cleaning up - -To cleanup the Kubernetes resources created by this tutorial, run: - -```bash -kubectl patch -n demo milvus/quick-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" -kubectl delete -n demo milvus/quick-milvus - -kubectl delete ns demo -``` diff --git a/docs/guides/milvus/tls/_index.md b/docs/guides/milvus/tls/_index.md deleted file mode 100644 index 0a6bca5d2..000000000 --- a/docs/guides/milvus/tls/_index.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Milvus TLS -menu: - docs_{{ .version }}: - identifier: milvus-tls - name: TLS - parent: milvus-guides - weight: 25 -menu_name: docs_{{ .version }} ---- diff --git a/docs/guides/milvus/tls/overview.md b/docs/guides/milvus/tls/overview.md deleted file mode 100644 index 979dff780..000000000 --- a/docs/guides/milvus/tls/overview.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Milvus TLS Overview -menu: - docs_{{ .version }}: - identifier: milvus-tls-overview - name: Overview - parent: milvus-tls - weight: 10 -menu_name: docs_{{ .version }} -section_menu_id: guides ---- - -# Milvus TLS - -This guide shows the main TLS considerations for a Milvus deployment. - -## Before You Begin - -- Install `cert-manager` in your cluster. -- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). - -## Configure TLS - -Milvus TLS setup is managed with cert-manager and KubeDB TLS settings. - -- Use a trusted issuer for server certificates. -- Rotate certificates in a controlled maintenance window. - -## Verify - -```bash -kubectl get milvus -n demo milvus-cluster -o yaml -kubectl get secret -n demo -``` - -## Next Steps - -- Test client connectivity with TLS enabled before routing production traffic. -- Document the certificate rotation process for your team. From 96a0f92397104d5ac957970c3327799c3329571c Mon Sep 17 00:00:00 2001 From: urmi Date: Tue, 5 May 2026 16:06:34 +0600 Subject: [PATCH 6/6] add concepts, configuration and monitoring docs Signed-off-by: urmi --- docs/guides/milvus/README.md | 71 +++ docs/guides/milvus/_index.md | 10 + docs/guides/milvus/concepts/_index.md | 10 + docs/guides/milvus/concepts/catalog.md | 70 +++ docs/guides/milvus/concepts/milvus.md | 548 ++++++++++++++++++ docs/guides/milvus/configuration/_index.md | 10 + .../milvus/configuration/using-config-file.md | 137 +++++ docs/guides/milvus/custom-rbac/_index.md | 10 + .../milvus/custom-rbac/using-custom-rbac.md | 160 +++++ docs/guides/milvus/monitoring/_index.md | 10 + docs/guides/milvus/monitoring/overview.md | 39 ++ .../monitoring/using-prometheus-operator.md | 344 +++++++++++ docs/guides/milvus/ops-request/_index.md | 10 + docs/guides/milvus/ops-request/overview.md | 29 + docs/guides/milvus/private-registry/_index.md | 10 + .../using-private-registry.md | 126 ++++ docs/guides/milvus/quickstart/_index.md | 10 + docs/guides/milvus/quickstart/quickstart.md | 158 +++++ docs/guides/milvus/tls/_index.md | 10 + docs/guides/milvus/tls/overview.md | 39 ++ 20 files changed, 1811 insertions(+) create mode 100644 docs/guides/milvus/README.md create mode 100644 docs/guides/milvus/_index.md create mode 100644 docs/guides/milvus/concepts/_index.md create mode 100644 docs/guides/milvus/concepts/catalog.md create mode 100644 docs/guides/milvus/concepts/milvus.md create mode 100644 docs/guides/milvus/configuration/_index.md create mode 100644 docs/guides/milvus/configuration/using-config-file.md create mode 100644 docs/guides/milvus/custom-rbac/_index.md create mode 100644 docs/guides/milvus/custom-rbac/using-custom-rbac.md create mode 100644 docs/guides/milvus/monitoring/_index.md create mode 100644 docs/guides/milvus/monitoring/overview.md create mode 100644 docs/guides/milvus/monitoring/using-prometheus-operator.md create mode 100644 docs/guides/milvus/ops-request/_index.md create mode 100644 docs/guides/milvus/ops-request/overview.md create mode 100644 docs/guides/milvus/private-registry/_index.md create mode 100644 docs/guides/milvus/private-registry/using-private-registry.md create mode 100644 docs/guides/milvus/quickstart/_index.md create mode 100644 docs/guides/milvus/quickstart/quickstart.md create mode 100644 docs/guides/milvus/tls/_index.md create mode 100644 docs/guides/milvus/tls/overview.md diff --git a/docs/guides/milvus/README.md b/docs/guides/milvus/README.md new file mode 100644 index 000000000..ef8652c30 --- /dev/null +++ b/docs/guides/milvus/README.md @@ -0,0 +1,71 @@ +--- +title: Milvus +menu: + docs_{{ .version }}: + identifier: milvus-readme + name: Milvus + parent: milvus-guides + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +url: /docs/{{ .version }}/guides/milvus/ +aliases: + - /docs/{{ .version }}/guides/milvus/README/ +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Overview + +KubeDB supports vector database deployment with Milvus using the `Milvus` CRD. + +## Supported Milvus Features + +| Features | Availability | +|----------------------------------|:------------:| +| Standalone provisioning | ✓ | +| Distributed provisioning | ✓ | +| Monitoring | ✓ | +| TLS | No | +| Ops Requests | No | + +## Example Milvus Manifest + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +``` + +## User Guide + +- [Quickstart Milvus](/docs/guides/milvus/quickstart/quickstart.md) with KubeDB operator. +- [Milvus CRD Concept](/docs/guides/milvus/concepts/milvus.md). +- [MilvusVersion CRD Concept](/docs/guides/milvus/concepts/catalog.md). +- [MilvusOpsRequest CRD Concept](/docs/guides/milvus/concepts/opsrequest.md). +- [RBAC Quickstart](/docs/guides/milvus/quickstart/rbac.md) +- [Private Registry](/docs/guides/milvus/private-registry/using-private-registry.md) +- [Custom RBAC](/docs/guides/milvus/custom-rbac/using-custom-rbac.md) +- [Custom Configuration](/docs/guides/milvus/configuration/using-config-file.md) +- [Monitoring](/docs/guides/milvus/monitoring/overview.md) for metrics collection guidance. +- [Builtin Prometheus Monitoring](/docs/guides/milvus/monitoring/using-builtin-prometheus.md) +- [Prometheus Operator Monitoring](/docs/guides/milvus/monitoring/using-prometheus-operator.md) \ No newline at end of file diff --git a/docs/guides/milvus/_index.md b/docs/guides/milvus/_index.md new file mode 100644 index 000000000..2b29e986f --- /dev/null +++ b/docs/guides/milvus/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus +menu: + docs_{{ .version }}: + identifier: milvus-guides + name: Milvus + parent: guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/_index.md b/docs/guides/milvus/concepts/_index.md new file mode 100644 index 000000000..f522969c1 --- /dev/null +++ b/docs/guides/milvus/concepts/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Concepts +menu: + docs_{{ .version }}: + identifier: milvus-concepts + name: Concepts + parent: milvus-guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/concepts/catalog.md b/docs/guides/milvus/concepts/catalog.md new file mode 100644 index 000000000..3fabc23ab --- /dev/null +++ b/docs/guides/milvus/concepts/catalog.md @@ -0,0 +1,70 @@ +--- +title: MilvusVersion CRD +menu: + docs_{{ .version }}: + identifier: milvus-catalog-concepts + name: MilvusVersion + parent: milvus-concepts-milvus + weight: 15 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# MilvusVersion + +## What is MilvusVersion + +`MilvusVersion` is the catalog CRD that defines the Milvus engine image and related metadata for KubeDB-managed Milvus deployments. + +KubeDB uses this CRD when resolving `Milvus.spec.version`. + +## MilvusVersion Specification + +```yaml +apiVersion: catalog.kubedb.com/v1alpha1 +kind: MilvusVersion +metadata: + name: 2.6.11 +spec: + db: + image: ghcr.io/appscode-images/milvus:2.6.11 + etcdVersion: v3.5.21 + securityContext: + runAsUser: 1000 + version: 2.6.11 +``` + +### metadata.name + +`metadata.name` is a required field that specifies the name of the `MilvusVersion` CR. You have to specify this name in `spec.version` field of [Milvus](/docs/guides/milvus/concepts/milvus.md) CR. + +We follow this convention for naming MilvusVersion CR: + +- Name format: `{Original Milvus image version}-{modification tag}` + +We use official Apache Milvus release tar files to build docker images for supporting Milvus versions and re-tag the image with v1, v2 etc. modification tag when there's any. An image with higher modification tag will have more features than the images with lower modification tag. Hence, it is recommended to use MilvusVersion CR with the highest modification tag to enjoy the latest features. + +### spec.version + +`spec.version` is a required field that specifies the original version of Milvus database that has been used to build the docker image specified in `spec.db.image` field. + +### spec.db.image + +`spec.db.image` is a required field that specifies the docker image which will be used to create PetSet by KubeDB operator to create expected Milvus database. + +### spec.etcdVersion + +`spec.etcdVersion` specifies the compatible Etcd version required by this Milvus release. + +### spec.deprecated + +`spec.deprecated` is an optional field that specifies whether the docker images specified here is supported by the current KubeDB operator. + +The default value of this field is `false`. If `spec.deprecated` is set to `true`, KubeDB operator will skip processing this CRD object and will add a event to the CRD object specifying that the DB version is deprecated. + +## Next Steps + +- Read the [Milvus CRD concept](/docs/guides/milvus/concepts/milvus.md). +- Run the [Milvus quickstart](/docs/guides/milvus/quickstart/quickstart.md). \ No newline at end of file diff --git a/docs/guides/milvus/concepts/milvus.md b/docs/guides/milvus/concepts/milvus.md new file mode 100644 index 000000000..06bc2fe8a --- /dev/null +++ b/docs/guides/milvus/concepts/milvus.md @@ -0,0 +1,548 @@ +--- +title: Milvus CRD +menu: + docs_{{ .version }}: + identifier: milvus-concepts-milvus + name: Milvus + parent: milvus-concepts + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Milvus + +## What is Milvus + +`Milvus` is a KubeDB `CustomResourceDefinition` used to deploy and manage Milvus vector databases. You only need to describe the desired database configuration in a `Milvus`object, and the KubeDB operator will create Kubernetes objects in the desired state for you. + +## Milvus Spec + +As with all other Kubernetes objects, a Milvus needs `apiVersion`, `kind`, and `metadata` fields. It also needs a `.spec` section. Below is an example of Milvus object. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster + namespace: kubedb +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + metaStorage: + externallyManaged: true + endpoints: + - http://etcdcluster-sample-0.etcdcluster-sample.default.svc.cluster.local:2379 + - http://etcdcluster-sample-1.etcdcluster-sample.default.svc.cluster.local:2379 + - http://etcdcluster-sample-2.etcdcluster-sample.default.svc.cluster.local:2379 + disableSecurity: false + authSecret: + name: "milvus-auth" + externallyManaged: true + configuration: + secretName: my-release-user-config + inline: + milvus.yaml: | + log: + level: info + file: + maxAge: 30 + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 600m + memory: 2Gi + + datanode: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 600m + memory: 1Gi + limits: + cpu: 700m + memory: 3Gi + + proxy: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 600m + memory: 4Gi + querynode: + replicas: 2 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 800m + memory: 3Gi + limits: + cpu: 900m + memory: 4Gi + streamingnode: + replicas: 3 + podTemplate: + spec: + containers: + - name: milvus + resources: + requests: + cpu: 600m + memory: 2Gi + limits: + cpu: 700m + memory: 2Gi + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 2Gi + monitor: + agent: prometheus.io/operator + prometheus: + exporter: + port: 9091 + resources: + limits: + memory: 512Mi + requests: + cpu: 600m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + serviceMonitor: + interval: 10s + labels: + release: prometheus + tls: + issuerRef: + name: milvus-issuer + kind: Issuer + apiGroup: "cert-manager.io" + external: + mode: mTLS + internal: + mode: TLS + deletionPolicy: WipeOut + healthChecker: + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 2 + disableWriteCheck: false +``` + +### spec.version + +`spec.version` is a required field specifying the name of the [MilvusVersion](/docs/guides/milvus/concepts/milvusversion.md) crd where the docker images are specified. Currently, when you install KubeDB, it creates the following `Milvus` resources, + +- `2.6.7` +- `2.6.9` +- `2.6.11` + +### spec.objectStorage + +`spec.objectStorage` is a required field that specifies the object storage backend used by Milvus. Milvus depends on external object storage (such as MinIO or any S3-compatible service) to store its data. + +The configuration is provided via a Kubernetes `Secret` referenced in this field: + +```yaml +objectStorage: + configSecret: + name: my-release-minio +``` +The referenced secret must contain the following keys: + +- address – endpoint of the object storage service +- accesskey – username for authentication +- secretkey – password for authentication + +All values must be base64-encoded. + +In this setup, MinIO is deployed separately (for example, via Helm) and acts as a dependency for Milvus. KubeDB does not manage MinIO directly; it only uses the credentials provided through the secret to connect to the object storage. + +### spec.metaStorage + +`spec.metaStorage` defines how the metadata store (etcd) used by Milvus is configured. Milvus relies on etcd to manage cluster metadata and coordination. The etcd operator must be installed and running in the user cluster. + +- `externallyManaged` indicates whether the etcd cluster is managed outside of KubeDB. + - If true, users must provide the etcd endpoints. + - If false, KubeDB will create and manage an EtcdCluster resource. + +- `endpoints` is the list of etcd client endpoints and required when externallyManaged: true. +- `size (optional)` is the number of etcd nodes to provision and used only when externallyManaged: false. +- `storageType (optional)` defines storage behavior (e.g., durable or ephemeral). +- `storage (optional)` specifies the PersistentVolume configuration for etcd when managed by KubeDB. + +### spec.authSecret + +`spec.authSecret` is an optional field that points to a Secret used to hold credentials for `milvus` root user. If not set, KubeDB operator creates a new Secret `{milvus-object-name}-auth` for storing the password for `root` user for each Milvus object. + +We can use this field in 3 mode. +1. Using an external secret. In this case, You need to create an auth secret first with required fields, then specify the secret name when creating the Milvus object using `spec.authSecret.name` & set `spec.authSecret.externallyManaged` to true. +```yaml +authSecret: + name: + externallyManaged: true +``` + +2. Specifying the secret name only. In this case, You need to specify the secret name when creating the Milvus object using `spec.authSecret.name`. `externallyManaged` is by default false. +```yaml +authSecret: + name: +``` + +3. Let KubeDB do everything for you. In this case, no work for you. + +AuthSecret contains a `user` key and a `password` key which contains the `username` and `password` respectively for Milvus `root` user. + +Example: + +```bash +$ kubectl create secret generic milvus-auth -n demo \ +--from-literal=username=root \ +--from-literal=password=6q8u_2jMOW-OOZXk +secret "milvus-auth" created +``` + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: milvus-auth + namespace: kubedb +type: kubernetes.io/basic-auth +stringData: + username: "root" + password: "Milvus" +``` + +Secrets provided by users are not managed by KubeDB, and therefore, won't be modified or garbage collected by the KubeDB operator (version 0.13.0 and higher). + +### spec.configuration +`spec.configuration` is an optional field that specifies custom configuration for Milvus cluster. It has the following fields: +- `configuration.secretName` is an optional field that specifies the name of the secret that holds custom configuration files for Milvus cluster. +- `configuration.inline` is an optional field that allows you to provide custom configuration directly in the Milvus object. + +```yaml +configuration: + secretName: my-release-user-config + inline: + milvus.yaml: | + log: + level: info + file: + maxAge: 30 +``` + +### spec.topology + +`spec.topology` defines the deployment topology for Milvus. It supports both **Standalone** and **Distributed** (cluster) modes. + +### spec.topology.mode + +Specifies the deployment mode of Milvus. + +- **`Standalone`**: Runs Milvus as a single-node deployment. All components run inside a single pod. +- **`Distributed`**: Runs Milvus as a multi-component distributed cluster. + +### spec.topology.standalone + +```yaml +topology: + mode: Standalone +``` +When `mode: Standalone` is used: +- All Milvus components run in a single unified deployment. +- No separate component configuration (like datanode, proxy, etc.) is required. +- KubeDB manages all internal services automatically + +### spec.topology.distributed + +`distributed` contains the configuration for all Milvus components in distributed mode. + +#### spec.topology.mixcoord + +`mixcoord` is responsible for coordinating metadata and internal cluster orchestration. + +- **replicas**: Number of mixcoord pods (default: `1`) +- **podTemplate**: Custom resource requests/limits and pod-level configuration + + +#### spec.topology.distributed.datanode + +`datanode` handles data ingestion and persistence into storage. + +- **replicas**: Number of datanode pods (default: `1`) +- **podTemplate**: Resource configuration for each datanode pod + +#### spec.topology.distributed.proxy + +`proxy` is the entry point for client requests. + +- **replicas**: Number of proxy pods (default: `1`) +- **podTemplate**: Resource configuration for proxy pods + +#### spec.topology.distributed.querynode + +`querynode` executes search and query operations on vector data. + +- **replicas**: Number of querynode pods (default: `1`) +- **podTemplate**: Resource configuration for query execution workload + + +#### spec.topology.distributed.streamingnode + +`streamingnode` handles real-time streaming ingestion. + +- **replicas**: Number of streamingnode pods (default: `1`) +- **podTemplate**: Resource configuration for streaming workloads + +Additional storage configuration for `streamingnode`: + +- **storageType**: Defines storage behavior (`Durable` or `Ephemeral`) +- **storage**: PVC specification used when `storageType` is `Durable` + +### spec..podTemplate + +KubeDB allows providing a template for database pod through `spec..podTemplate`. KubeDB operator will pass the information provided in `spec..podTemplate` to the PetSet created for Milvus cluster. + +KubeDB accept following fields to set in `spec..podTemplate:` + +- metadata: + - annotations (pod's annotation) + - labels (pod's labels) +- controller: + - annotations (petset's annotation) + - labels (petset's labels) +- spec: + - containers + - volumes + - podPlacementPolicy + - initContainers + - containers + - imagePullSecrets + - nodeSelector + - serviceAccountName + - schedulerName + - tolerations + - priorityClassName + - priority + - securityContext + +You can check out the full list [here](https://github.com/kmodules/offshoot-api/blob/master/api/v2/types.go#L26C1-L279C1). +Uses of some field of `spec..podTemplate` is described below, + +#### spec..podTemplate.spec.tolerations + +The `spec.podTemplate.spec.tolerations` is an optional field. This can be used to specify the pod's tolerations. + +#### spec..podTemplate.spec.volumes + +The `spec..podTemplate..volumes` is an optional field. This can be used to provide the list of volumes that can be mounted by containers belonging to the pod. + +#### spec..podTemplate.spec.podPlacementPolicy + +`spec.podTemplate.spec.podPlacementPolicy` is an optional field. This can be used to provide the reference of the `podPlacementPolicy`. `name` of the podPlacementPolicy is referred under this attribute. This will be used by our Petset controller to place the db pods throughout the region, zone & nodes according to the policy. It utilizes kubernetes affinity & podTopologySpreadContraints feature to do so. +```yaml +spec: + podPlacementPolicy: + name: default +``` + +#### spec..podTemplate.spec.nodeSelector + +`spec..podTemplate.spec.nodeSelector` is an optional field that specifies a map of key-value pairs. For the pod to be eligible to run on a node, the node must have each of the indicated key-value pairs as labels (it can have additional labels as well). To learn more, see [here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector) . + +#### spec..podTemplate.spec.containers + +The `spec..podTemplate.spec.containers` can be used to provide the list containers and their configurations for to the database pod. some of the fields are described below, + +##### spec..podTemplate.spec.containers[].name +The `spec..podTemplate.spec.containers[].name` field used to specify the name of the container specified as a DNS_LABEL. Each container in a pod must have a unique name (DNS_LABEL). Cannot be updated. + +##### spec..podTemplate.spec.containers[].args +`spec..podTemplate.spec.containers[].args` is an optional field. This can be used to provide additional arguments to database installation. + +##### spec..podTemplate.spec.containers[].env + +`spec..podTemplate.spec.containers[].env` is an optional field that specifies the environment variables to pass to the Redis containers. + +##### spec..podTemplate.spec.containers[].resources + +`spec..podTemplate.spec.containers[].resources` is an optional field. This can be used to request compute resources required by containers of the database pods. To learn more, visit [here](http://kubernetes.io/docs/user-guide/compute-resources/). + +### spec.serviceTemplates + +You can also provide template for the services created by KubeDB operator for Milvus cluster through `spec.serviceTemplates`. This will allow you to set the type and other properties of the services. + +KubeDB allows following fields to set in `spec.serviceTemplates`: +- `alias` represents the identifier of the service. It has the following possible value: + - `stats` for is used for the `exporter` service identification. + +Milvus comes with one primary services used for client access (Standalone or Proxy in Distributed mode) and four component services for distributed mode (`mixcoord`, `datanode`, `querynode` and `streamingnode`). There are two options for providing serviceTemplates: +- To provide `serviceTemplates` for a specific service, the `serviceTemplates.ports.port` should be equal to the port of that service and `serviceTemplate` will be used for that particular service only. +- However, to provide a common `serviceTemplates`, `serviceTemplates.ports.port` should be empty. + +- metadata: + - labels + - annotations +- spec: + - type + - ports + - clusterIP + - externalIPs + - loadBalancerIP + - loadBalancerSourceRanges + - externalTrafficPolicy + - healthCheckNodePort + - sessionAffinityConfig + +See [here](https://github.com/kmodules/offshoot-api/blob/kubernetes-1.21.1/api/v1/types.go#L237) to understand these fields in detail. + +### spec.monitor + +Milvus managed by KubeDB can be monitored with Prometheus operator out-of-the-box. To learn more, +- [Monitor Apache Milvus with Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md) + +### spec.tls + +`spec.tls` defines the TLS configuration for securing Milvus communication. The KubeDB operator uses [cert-manager](https://cert-manager.io/) to issue and manage certificates. Currently, only **PKCS#8 encoded certificates** are supported. + +TLS in Milvus can be configured for: +- **External traffic** (client → Milvus) +- **Internal traffic** (inter-component communication) + +--- + +### Example + +```yaml +spec: + tls: + issuerRef: + apiGroup: cert-manager.io + kind: Issuer + name: milvus-issuer + external: + mode: mTLS + internal: + mode: TLS +``` +The `spec.tls` contains the following fields: + +- `tls.issuerRef` - is an `optional` field that references to the `Issuer` or `ClusterIssuer` custom resource object of [cert-manager](https://cert-manager.io/docs/concepts/issuer/). It is used to generate the necessary certificate secrets for Milvus. If the `issuerRef` is not specified, the operator creates a self-signed CA and also creates necessary certificate (valid: 365 days) secrets using that CA. + - `apiGroup` - is the group name of the resource that is being referenced. Currently, the only supported value is `cert-manager.io`. + - `kind` - is the type of resource that is being referenced. The supported values are `Issuer` and `ClusterIssuer`. + - `name` - is the name of the resource ( `Issuer` or `ClusterIssuer` ) that is being referenced. + + +- `tls.external` - `external` controls TLS for client-facing traffic (gRPC / REST). + - `TLS` - requires only the server certificate to encrypt communication between client and Milvus proxy. + - `mTLS` - requires both server and client certificates to enable mutual authentication between client and Milvus proxy. + + +- `tls.internal` - `internal` enables TLS for inter-component communication within the Milvus cluster. + - `TLS` - uses only server-side certificates to encrypt communication between internal Milvus components (e.g., proxy, querynode, datanode). + - `mTLS` - not supported for internal communication; internal traffic is generally secured using one-way TLS only. + + +- `tls.certificates` - is an `optional` field that specifies a list of certificate configurations used to configure the certificates. It has the following fields: + - `alias` - represents the identifier of the certificate. It has the following possible value: + - `server` - is used for the server certificate configuration. + - `client` - is used for the client certificate configuration. + + + - `secretName` - ( `string` | `"-alias-cert"` ) - specifies the k8s secret name that holds the certificates. + + + - `subject` - specifies an `X.509` distinguished name (DN). It has the following configurable fields: + - `organizations` ( `[]string` | `nil` ) - is a list of organization names. + - `organizationalUnits` ( `[]string` | `nil` ) - is a list of organization unit names. + - `countries` ( `[]string` | `nil` ) - is a list of country names (ie. Country Codes). + - `localities` ( `[]string` | `nil` ) - is a list of locality names. + - `provinces` ( `[]string` | `nil` ) - is a list of province names. + - `streetAddresses` ( `[]string` | `nil` ) - is a list of street addresses. + - `postalCodes` ( `[]string` | `nil` ) - is a list of postal codes. + - `serialNumber` ( `string` | `""` ) is a serial number. + + For more details, visit [here](https://golang.org/pkg/crypto/x509/pkix/#Name). + + + - `duration` ( `string` | `""` ) - is the period during which the certificate is valid. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as `"300m"`, `"1.5h"` or `"20h45m"`. Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". + - `renewBefore` ( `string` | `""` ) - is a specifiable time before expiration duration. + - `dnsNames` ( `[]string` | `nil` ) - is a list of subject alt names. + - `ipAddresses` ( `[]string` | `nil` ) - is a list of IP addresses. + - `uris` ( `[]string` | `nil` ) - is a list of URI Subject Alternative Names. + - `emailAddresses` ( `[]string` | `nil` ) - is a list of email Subject Alternative Names. + +### spec.deletionPolicy + +`deletionPolicy` gives flexibility whether to `nullify`(reject) the delete operation of `Milvus` crd or which resources KubeDB should keep or delete when you delete `Milvus` crd. KubeDB provides following four deletion policies: + +- DoNotTerminate +- WipeOut +- Halt +- Delete + +When `deletionPolicy` is `DoNotTerminate`, KubeDB takes advantage of `ValidationWebhook` feature in Kubernetes 1.9.0 or later clusters to implement `DoNotTerminate` feature. If admission webhook is enabled, `DoNotTerminate` prevents users from deleting the database as long as the `spec.deletionPolicy` is set to `DoNotTerminate`. + +> For more details you can visit [here](https://appscode.com/blog/post/deletion-policy/) + +### spec.healthChecker +It defines the attributes for the health checker. +- `spec.healthChecker.periodSeconds` specifies how often to perform the health check. +- `spec.healthChecker.timeoutSeconds` specifies the number of seconds after which the probe times out. +- `spec.healthChecker.failureThreshold` specifies minimum consecutive failures for the healthChecker to be considered failed. +- `spec.healthChecker.disableWriteCheck` specifies whether to disable the writeCheck or not. + +Know details about KubeDB Health checking from this [blog post](https://appscode.com/blog/post/kubedb-health-checker/). + +## Next Steps + +- Learn how to use KubeDB to run Apache Milvus cluster [here](/docs/guides/milvus/README.md). +- Deploy [dedicated topology cluster](/docs/guides/milvus/clustering/guide/index.md) for Apache Milvus +- Monitor your Milvus cluster with KubeDB using [`out-of-the-box` Prometheus operator](/docs/guides/milvus/monitoring/using-prometheus-operator.md). +- Detail concepts of [MilvusVersion object](/docs/guides/milvus/concepts/milvusversion.md). + +[//]: # (- Learn to use KubeDB managed Milvus objects using [CLIs](/docs/guides/milvus/cli/cli.md).) +- Want to hack on KubeDB? Check our [contribution guidelines](/docs/CONTRIBUTING.md). \ No newline at end of file diff --git a/docs/guides/milvus/configuration/_index.md b/docs/guides/milvus/configuration/_index.md new file mode 100644 index 000000000..0224bf33d --- /dev/null +++ b/docs/guides/milvus/configuration/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus with Custom Configuration +menu: + docs_{{ .version }}: + identifier: milvus-configuration + name: Custom Configuration + parent: milvus-guides + weight: 130 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/configuration/using-config-file.md b/docs/guides/milvus/configuration/using-config-file.md new file mode 100644 index 000000000..1810929b3 --- /dev/null +++ b/docs/guides/milvus/configuration/using-config-file.md @@ -0,0 +1,137 @@ +--- +title: Run Milvus with Custom Configuration +menu: + docs_{{ .version }}: + identifier: milvus-using-config-file + name: Config File + parent: milvus-configuration + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Custom Configuration File + +KubeDB supports providing custom configuration for Milvus. This tutorial will show you how to use KubeDB to run Milvus with custom configuration. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Overview + +Milvus supports configuration via the `milvus.yaml` file. KubeDB takes advantage of `spec.configuration.secretName` to allow users to provide a custom `milvus.yaml` without mounting any volume into the Pod. The operator reads this Secret internally and applies the configuration automatically. + +In this tutorial, we will configure `queryNode.gracefulTime` and `dataNode.segment.maxSize` parameters. + +## Custom Configuration + +At first, let's create a custom `milvus.yaml` file: + +```yaml +queryNode: + gracefulTime: 5000 + +dataNode: + segment: + maxSize: 512 +``` + +Now, create a Secret with this configuration file. + +```bash +$ kubectl create secret generic -n demo milvus-configuration \ + --from-file=milvus.yaml=./milvus.yaml +secret/milvus-configuration created +``` + +Verify the Secret has the configuration file. + +```bash +$ kubectl get secret -n demo milvus-configuration -o yaml +apiVersion: v1 +data: + milvus.yaml: +kind: Secret +metadata: + name: milvus-configuration + namespace: demo +``` + +Now, create Milvus CRD specifying `spec.configuration.secretName` field. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: custom-milvus + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + configuration: + secretName: milvus-configuration + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/configuration/milvus-configuration.yaml +milvus.kubedb.com/custom-milvus created +``` + +Now, wait for the Milvus to be ready. + +```bash +$ kubectl get milvus -n demo custom-milvus +NAME VERSION STATUS AGE +custom-milvus 2.6.11 Ready 3m +``` + +Check that the pod is running: + +```bash +$ kubectl get pod -n demo custom-milvus-0 +NAME READY STATUS RESTARTS AGE +custom-milvus-0 1/1 Running 0 3m +``` + +Now, we will verify the configuration has been applied. We will `exec` into the pod and check the configuration file. + +```bash +$ kubectl exec -it -n demo custom-milvus-0 -- cat /milvus/configs/milvus.yaml | grep -A 5 queryNode +queryNode: + gracefulTime: 5000 +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/custom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/custom-milvus + +kubectl delete -n demo secret milvus-configuration +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/custom-rbac/_index.md b/docs/guides/milvus/custom-rbac/_index.md new file mode 100644 index 000000000..d322e1d6d --- /dev/null +++ b/docs/guides/milvus/custom-rbac/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus with Custom RBAC resources +menu: + docs_{{ .version }}: + identifier: milvus-custom-rbac + name: Custom RBAC + parent: milvus-guides + weight: 90 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/custom-rbac/using-custom-rbac.md b/docs/guides/milvus/custom-rbac/using-custom-rbac.md new file mode 100644 index 000000000..a5ee49c75 --- /dev/null +++ b/docs/guides/milvus/custom-rbac/using-custom-rbac.md @@ -0,0 +1,160 @@ +--- +title: Run Milvus with Custom RBAC resources +menu: + docs_{{ .version }}: + identifier: milvus-custom-rbac-quickstart + name: Custom RBAC + parent: milvus-custom-rbac + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Custom RBAC Resources + +KubeDB supports finer user control over role based access permissions provided to a Milvus instance. This tutorial will show you how to use KubeDB to run Milvus instance with custom RBAC resources. + +## Before You Begin + +At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +## Overview + +KubeDB allows users to provide custom RBAC resources, namely, `ServiceAccount`, `Role`, and `RoleBinding` for Milvus. This is provided via the `spec.podTemplate.spec.serviceAccountName` field in Milvus CRD. + +## Custom RBAC for Milvus + +At first, let's create a `Service Account` in `demo` namespace. + +```bash +$ kubectl create serviceaccount -n demo my-custom-serviceaccount +serviceaccount/my-custom-serviceaccount created +``` + +Now, we need to create a role that has necessary access permissions for the Milvus database named `quick-milvus`. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: my-custom-role + namespace: demo +rules: +- apiGroups: + - apps + resourceNames: + - quick-milvus + resources: + - petsets + verbs: + - get +- apiGroups: + - kubedb.com + resourceNames: + - quick-milvus + resources: + - milvuses + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - list + - patch +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-role.yaml +role.rbac.authorization.k8s.io/my-custom-role created +``` + +Now create a `RoleBinding` to bind this `Role` with the already created service account. + +```bash +$ kubectl create rolebinding my-custom-rolebinding \ + --role=my-custom-role \ + --serviceaccount=demo:my-custom-serviceaccount \ + --namespace=demo +rolebinding.rbac.authorization.k8s.io/my-custom-rolebinding created +``` + +Now, create a Milvus CRD specifying `spec.podTemplate.spec.serviceAccountName` field to `my-custom-serviceaccount`. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: quick-milvus + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + storageType: Durable + podTemplate: + spec: + serviceAccountName: my-custom-serviceaccount + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + deletionPolicy: WipeOut +``` + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/custom-rbac/milvus-custom-db.yaml +milvus.kubedb.com/quick-milvus created +``` + +Check that the pod is running: + +```bash +$ kubectl get pod -n demo quick-milvus-0 +NAME READY STATUS RESTARTS AGE +quick-milvus-0 1/1 Running 0 3m +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/quick-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/quick-milvus + +kubectl delete -n demo serviceaccount my-custom-serviceaccount +kubectl delete -n demo role my-custom-role +kubectl delete -n demo rolebinding my-custom-rolebinding +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/monitoring/_index.md b/docs/guides/milvus/monitoring/_index.md new file mode 100644 index 000000000..a1c83edfb --- /dev/null +++ b/docs/guides/milvus/monitoring/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Monitoring +menu: + docs_{{ .version }}: + identifier: milvus-monitoring + name: Monitoring + parent: milvus-guides + weight: 20 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/monitoring/overview.md b/docs/guides/milvus/monitoring/overview.md new file mode 100644 index 000000000..248796e32 --- /dev/null +++ b/docs/guides/milvus/monitoring/overview.md @@ -0,0 +1,39 @@ +--- +title: Milvus Monitoring Overview +menu: + docs_{{ .version }}: + identifier: milvus-monitoring-overview + name: Overview + parent: milvus-monitoring + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus Monitoring + +This guide shows how to enable and verify monitoring for Milvus deployments. + +## Before You Begin + +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). +- Install Prometheus Operator or another compatible metrics collection stack. + +## Enable Monitoring + +Milvus supports monitoring integration through `spec.monitor`. + +- Configure monitoring agent and service monitor selectors. +- Validate scrape targets for proxy and distributed components. + +## Verify + +```bash +kubectl get milvus -n demo milvus-cluster -o yaml +kubectl get servicemonitor -A +``` + +## Next Steps + +- Review which Milvus components need dedicated scrape targets in distributed mode. +- Add dashboards for latency, indexing throughput, and node health. diff --git a/docs/guides/milvus/monitoring/using-prometheus-operator.md b/docs/guides/milvus/monitoring/using-prometheus-operator.md new file mode 100644 index 000000000..ac5487e9d --- /dev/null +++ b/docs/guides/milvus/monitoring/using-prometheus-operator.md @@ -0,0 +1,344 @@ +--- +title: Monitor Milvus using Prometheus Operator +menu: + docs_{{ .version }}: + identifier: milvus-using-prometheus-operator-monitoring + name: Prometheus Operator + parent: milvus-monitoring + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring Milvus using Prometheus Operator + +[Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) provides simple and Kubernetes native way to deploy and configure Prometheus, Alertmanager and related monitoring components. This tutorial will show you how to use the Prometheus operator to monitor Milvus database deployed with KubeDB. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/milvus/monitoring/overview.md). + +- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy the database in `demo` namespace. + + ```bash + $ kubectl create ns monitoring + namespace/monitoring created + + $ kubectl create ns demo + namespace/demo created + ``` + +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Find out required labels for ServiceMonitor + +We need to know the labels used to select `ServiceMonitor` by a `Prometheus` crd. We are going to provide these labels in `spec.monitor.prometheus.serviceMonitor.labels` field of Milvus crd so that KubeDB creates `ServiceMonitor` object accordingly. + +At first, let's find out the available Prometheus server in our cluster. + +```bash +$ kubectl get prometheus --all-namespaces +NAMESPACE NAME VERSION DESIRED READY RECONCILED AVAILABLE AGE +monitoring prometheus-kube-prometheus-prometheus v3.11.3 1 1 True True 25h +``` + +> If you don't have any Prometheus server running in your cluster, deploy one following the guide specified in **Before You Begin** section. + +Now, let's view the YAML of the available Prometheus server `prometheus-kube-prometheus-prometheus` in `monitoring` namespace. + +```bash +$ kubectl get prometheus -n monitoring prometheus-kube-prometheus-prometheus -oyaml +``` +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + annotations: + meta.helm.sh/release-name: prometheus + meta.helm.sh/release-namespace: monitoring + creationTimestamp: "2026-05-04T04:56:27Z" + generation: 1 + labels: + app: kube-prometheus-stack-prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/part-of: kube-prometheus-stack + app.kubernetes.io/version: 84.5.0 + chart: kube-prometheus-stack-84.5.0 + heritage: Helm + release: prometheus + name: prometheus-kube-prometheus-prometheus + namespace: monitoring + resourceVersion: "404630" + uid: 6cf79306-5f52-46e7-9342-b7447016823e +spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - prometheus + - key: app.kubernetes.io/instance + operator: In + values: + - prometheus-kube-prometheus-prometheus + topologyKey: kubernetes.io/hostname + weight: 100 + alerting: + alertmanagers: + - apiVersion: v2 + name: prometheus-kube-prometheus-alertmanager + namespace: monitoring + pathPrefix: / + port: http-web + automountServiceAccountToken: true + enableAdminAPI: false + enableOTLPReceiver: false + evaluationInterval: 30s + externalUrl: http://prometheus-kube-prometheus-prometheus.monitoring:9090 + hostNetwork: false + image: quay.io/prometheus/prometheus:v3.11.3 + imagePullPolicy: IfNotPresent + listenLocal: false + logFormat: logfmt + logLevel: info + paused: false + podMonitorNamespaceSelector: {} + podMonitorSelector: + matchLabels: + release: prometheus + portName: http-web + probeNamespaceSelector: {} + probeSelector: + matchLabels: + release: prometheus + replicas: 1 + retention: 10d + routePrefix: / + ruleNamespaceSelector: {} + ruleSelector: + matchLabels: + release: prometheus + scrapeConfigNamespaceSelector: {} + scrapeConfigSelector: + matchLabels: + release: prometheus + scrapeInterval: 30s + securityContext: + fsGroup: 2000 + runAsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + serviceAccountName: prometheus-kube-prometheus-prometheus + serviceMonitorNamespaceSelector: {} + serviceMonitorSelector: + matchLabels: + release: prometheus + shards: 1 + tsdb: + outOfOrderTimeWindow: 0s + version: v3.11.3 + walCompression: true +status: + availableReplicas: 1 + conditions: + - lastTransitionTime: "2026-05-04T04:57:09Z" + message: "" + observedGeneration: 1 + reason: "" + status: "True" + type: Available + - lastTransitionTime: "2026-05-04T04:56:37Z" + message: "" + observedGeneration: 1 + reason: "" + status: "True" + type: Reconciled + paused: false + replicas: 1 + selector: app.kubernetes.io/instance=prometheus-kube-prometheus-prometheus,app.kubernetes.io/managed-by=prometheus-operator,app.kubernetes.io/name=prometheus,operator.prometheus.io/name=prometheus-kube-prometheus-prometheus,prometheus=prometheus-kube-prometheus-prometheus + shardStatuses: + - availableReplicas: 1 + replicas: 1 + shardID: "0" + unavailableReplicas: 0 + updatedReplicas: 1 + shards: 1 + unavailableReplicas: 0 + updatedReplicas: 1 +``` +Notice the `spec.serviceMonitorSelector` section. Here, `release: prometheus` label is used to select `ServiceMonitor` CR. So, we are going to use this label in `spec.monitor.prometheus.labels` field of Milvus CR. + + +## Deploy Milvus with Monitoring Enabled + +At first, let's deploy a Milvus database with monitoring enabled. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: coreos-prom-milvus + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + deletionPolicy: WipeOut + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + interval: 10s + labels: + release: prometheus + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +``` + +Here, +- `spec.monitor.agent: prometheus.io/operator` indicates that we are going to monitor this server using Prometheus operator. +- `spec.monitor.prometheus.serviceMonitor.labels` specifies that KubeDB should create `ServiceMonitor` with these labels. +- `spec.monitor.prometheus.serviceMonitor.interval` indicates that the Prometheus should scrape metrics from this database with 10 seconds interval. + +Let's create the Milvus object that we have shown above, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/monitoring/coreos-prom-milvus.yaml +milvus.kubedb.com/coreos-prom-milvus created +``` + +Now, wait for the database to go into `Ready` state. + +```bash +$ kubectl get milvus -n demo coreos-prom-milvus +NAME VERSION STATUS AGE +coreos-prom-milvus 2.6.11 Ready 2m +``` +KubeDB will create a separate stats service with name `{Milvus crd name}-stats` for monitoring purpose. + +```bash +$ kubectl get svc -n kubedb --selector="app.kubernetes.io/instance=coreos-prom-milvus" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +coreos-prom-milvus-stats ClusterIP 10.43.113.92 9091/TCP 50m +``` +Here, `coreos-prom-milvus-stats` service has been created for monitoring purpose. +Let's describe this stats service. + +```bash +$ kubectl describe svc -n demo coreos-prom-milvus-stats +``` +```yaml +Name: coreos-prom-milvus-stats +Namespace: kubedb +Labels: app.kubernetes.io/component=database + app.kubernetes.io/instance=coreos-prom-milvus-stats + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=milvuses.kubedb.com + kubedb.com/role=stats +Annotations: monitoring.appscode.com/agent: prometheus.io/operator +Selector: app.kubernetes.io/instance=coreos-prom-milvus-stats,app.kubernetes.io/managed-by=kubedb.com,app.kubernetes.io/name=milvuses.kubedb.com +Type: ClusterIP +IP Family Policy: SingleStack +IP Families: IPv4 +IP: 10.43.113.92 +IPs: 10.43.113.92 +Port: metrics 9091/TCP +TargetPort: metrics/TCP +Endpoints: 10.42.0.102:9091,10.42.0.107:9091,10.42.0.111:9091 + 2 more... +Session Affinity: None +Events: +``` +Notice the `Labels` and `Port` fields. `ServiceMonitor` will use this information to target its endpoints. + +KubeDB will also create a `ServiceMonitor` crd in `demo` namespace that select the endpoints of `coreos-prom-milvus-stats` service. Verify that the `ServiceMonitor` crd has been created. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +coreos-prom-milvus-stats 2m +``` + + +Let's verify the `ServiceMonitor` has the right label to be discovered by Prometheus: + +```bash +$ kubectl get servicemonitor -n demo coreos-prom-milvus-stats -o yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + release: prometheus + name: coreos-prom-milvus-stats + namespace: demo +spec: + endpoints: + - honorLabels: true + interval: 10s + path: /metrics + port: metrics + namespaceSelector: + matchNames: + - demo + selector: + matchLabels: + app.kubernetes.io/instance: coreos-prom-milvus +``` +Notice that the `ServiceMonitor` has label `release: prometheus` that we had specified in Milvus crd. + +Also notice that the `ServiceMonitor` has selector which match the labels we have seen in the `coreos-prom-milvus-stats` service. It also, target the `metrics` port that we have seen in the stats service. + +## Verify Monitoring Metrics + +At first, let's find out the respective Prometheus pod for `prometheus` Prometheus server. + +```bash +$ kubectl get pod -n monitoring -l=app.kubernetes.io/name=prometheus +NAME READY STATUS RESTARTS AGE +prometheus-prometheus-kube-prometheus-prometheus-0 2/2 Running 0 27h +``` + +Prometheus server is listening to port `9090` of `prometheus-prometheus-kube-prometheus-prometheus-0` pod. We are going to use [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) to access Prometheus dashboard. + +Run following command on a separate terminal to forward the port 9090 of `prometheus-prometheus-kube-prometheus-prometheus-0` pod, + +```bash +$ kubectl port-forward -n monitoring prometheus-prometheus-kube-prometheus-prometheus-0 9090 +Forwarding from 127.0.0.1:9090 -> 9090 +Forwarding from [::1]:9090 -> 9090 +``` + +Now, we can access the dashboard at `localhost:9090`. Open [http://localhost:9090](http://localhost:9090) in your browser. You should see `metrics` endpoint of `coreos-prom-milvus-stats` service as one of the targets. + +Check the `endpoint` and `service` labels marked by the red rectangles. It verifies that the target is our expected database. Now, you can view the collected metrics and create a graph from homepage of this Prometheus dashboard. You can also use this Prometheus server as data source for [Grafana](https://grafana.com/) and create a beautiful dashboard with collected metrics. + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/coreos-prom-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/coreos-prom-milvus + +kubectl delete ns demo +kubectl delete ns monitoring +``` diff --git a/docs/guides/milvus/ops-request/_index.md b/docs/guides/milvus/ops-request/_index.md new file mode 100644 index 000000000..9d0b5d697 --- /dev/null +++ b/docs/guides/milvus/ops-request/_index.md @@ -0,0 +1,10 @@ +--- +title: Ops Request +menu: + docs_{{ .version }}: + identifier: milvus-ops-request + name: Ops Request + parent: milvus-guides + weight: 30 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/ops-request/overview.md b/docs/guides/milvus/ops-request/overview.md new file mode 100644 index 000000000..ae0b7e728 --- /dev/null +++ b/docs/guides/milvus/ops-request/overview.md @@ -0,0 +1,29 @@ +--- +title: Milvus Ops Request Overview +menu: + docs_{{ .version }}: + identifier: milvus-ops-request-overview + name: Overview + parent: milvus-ops-request + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus Ops Request + +This page tracks Milvus operations supported by the new database matrix used for these guides. + +## Before You Begin + +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). +- Confirm the current support list in [new_db.md](/new_db.md) for your release. + +## Current Status + +Based on [new_db.md](/new_db.md), no Milvus ops request types are listed yet. + +## What to Check Next + +- Watch future KubeDB releases for Milvus `OpsRequest` support. +- Use the TLS and monitoring guides for common day-2 platform tasks. diff --git a/docs/guides/milvus/private-registry/_index.md b/docs/guides/milvus/private-registry/_index.md new file mode 100644 index 000000000..c202e6450 --- /dev/null +++ b/docs/guides/milvus/private-registry/_index.md @@ -0,0 +1,10 @@ +--- +title: Run Milvus using Private Registry +menu: + docs_{{ .version }}: + identifier: milvus-private-registry + name: Private Registry + parent: milvus-guides + weight: 120 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/private-registry/using-private-registry.md b/docs/guides/milvus/private-registry/using-private-registry.md new file mode 100644 index 000000000..3ebbb5ab4 --- /dev/null +++ b/docs/guides/milvus/private-registry/using-private-registry.md @@ -0,0 +1,126 @@ +--- +title: Run Milvus using Private Registry +menu: + docs_{{ .version }}: + identifier: milvus-using-private-registry + name: Quickstart + parent: milvus-private-registry + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Using Private Docker Registry + +KubeDB supports using private Docker registries. This tutorial will show you how to run KubeDB managed Milvus database using private Docker images. + +## Before You Begin + +At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create ns demo +namespace/demo created +``` + +## Prepare Private Docker Registry + +- You will also need a docker private [registry](https://docs.docker.com/registry/) or [private repository](https://docs.docker.com/docker-hub/repos/#private-repositories). + +- Push the required images from KubeDB's [Docker hub account](https://hub.docker.com/r/kubedb/) into your private registry. For Milvus, push the `DB_IMAGE` of the following MilvusVersions, where `deprecated` is not true. + + ```bash + $ kubectl get milvusversions -o=custom-columns=NAME:.metadata.name,VERSION:.spec.version,DB_IMAGE:.spec.db.image,DEPRECATED:.spec.deprecated + NAME VERSION DB_IMAGE DEPRECATED + 2.4.0 2.4.0 kubedb/milvus:2.4.0 + ``` + +## Create ImagePullSecret + +Run the following command to create an image pull secret for your private Docker registry: + +```bash +$ kubectl create secret docker-registry -n demo myregistrykey \ + --docker-server=DOCKER_REGISTRY_SERVER \ + --docker-username=DOCKER_USER \ + --docker-email=DOCKER_EMAIL \ + --docker-password=DOCKER_PASSWORD +secret/myregistrykey created +``` + +## Install KubeDB Operator + +Follow the steps to [install KubeDB operator](/docs/setup/README.md) properly in cluster so that it points to the DOCKER_REGISTRY you wish to pull images from. + +## Create MilvusVersion CRD + +Create a MilvusVersion CRD specifying images from your private registry. Replace `PRIVATE_REGISTRY` with your private registry. + +```yaml +apiVersion: catalog.kubedb.com/v1alpha1 +kind: MilvusVersion +metadata: + name: "2.4.0" +spec: + db: + image: PRIVATE_REGISTRY/milvus:2.4.0 + version: "2.4.0" +``` + +```bash +$ kubectl apply -f pvt-milvusversion.yaml +milvusversion.catalog.kubedb.com/2.4.0 created +``` + +## Deploy Milvus from Private Registry + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: pvt-reg-milvus + namespace: demo +spec: + version: "2.4.0" + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + podTemplate: + spec: + imagePullSecrets: + - name: myregistrykey + deletionPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/private-registry/pvt-reg-milvus.yaml +milvus.kubedb.com/pvt-reg-milvus created +``` + +Check that the Milvus is in Running state: + +```bash +$ kubectl get pods -n demo --selector="app.kubernetes.io/instance=pvt-reg-milvus" +NAME READY STATUS RESTARTS AGE +pvt-reg-milvus-0 1/1 Running 0 3m +``` + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run: + +```bash +kubectl patch -n demo milvus/pvt-reg-milvus -p '{"spec":{"deletionPolicy":"WipeOut"}}' --type="merge" +kubectl delete -n demo milvus/pvt-reg-milvus + +kubectl delete ns demo +``` diff --git a/docs/guides/milvus/quickstart/_index.md b/docs/guides/milvus/quickstart/_index.md new file mode 100644 index 000000000..753aefff9 --- /dev/null +++ b/docs/guides/milvus/quickstart/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus Quickstart +menu: + docs_{{ .version }}: + identifier: milvus-quickstart + name: Quickstart + parent: milvus-guides + weight: 15 +menu_name: docs_{{ .version }} +--- \ No newline at end of file diff --git a/docs/guides/milvus/quickstart/quickstart.md b/docs/guides/milvus/quickstart/quickstart.md new file mode 100644 index 000000000..9078abdab --- /dev/null +++ b/docs/guides/milvus/quickstart/quickstart.md @@ -0,0 +1,158 @@ +--- +title: Milvus Quickstart +menu: + docs_{{ .version }}: + identifier: milvus-quickstart-overview + name: Overview + parent: milvus-quickstart + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Milvus QuickStart + +This tutorial shows how to run a Milvus database with KubeDB. + +

+  lifecycle +

+ +> Note: YAML files used in this tutorial are stored in [docs/examples/milvus/quickstart](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/examples/milvus/quickstart). + +## Before You Begin + +At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +Now, install KubeDB cli on your workstation and KubeDB operator in your cluster following the steps [here](/docs/setup/README.md) and make sure to include the flags `--set global.featureGates.Milvus=true` to ensure **Milvus CRD**. + +To keep things isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. + +```bash +$ kubectl create namespace demo +namespace/demo created + +$ kubectl get namespace +NAME STATUS AGE +demo Active 9s +``` + +## Check Available MilvusVersion + +When you install the KubeDB operator, it registers a CRD named [MilvusVersion](/docs/guides/milvus/concepts/catalog.md). The installation process comes with a set of tested MilvusVersion objects. Let's check available MilvusVersions by, + +```bash +$ kubectl get milvusversions +NAME VERSION DB_IMAGE DEPRECATED AGE +2.6.11 2.6.11 ghcr.io/appscode-images/milvus:2.6.11 6d2h +2.6.7 2.6.7 ghcr.io/appscode-images/milvus:2.6.7 6d2h +2.6.9 2.6.9 ghcr.io/appscode-images/milvus:2.6.9 6d2h +``` + +Notice the `DEPRECATED` column. Here, `true` means that this MilvusVersion is deprecated for the current KubeDB version. KubeDB will not work for deprecated MilvusVersion. You can also use the short from `mvversion` to check available MilvusVersions. + +In this tutorial, we will use `2.6.11` MilvusVersion CR to create a Milvus cluster. + +## Get External Dependencies Ready + +### Object Storage + +One of the external dependency of Milvus is object storage where the segments are stored. It is a storage mechanism that Milvus does not provide. **S3-compatible storage** (like **Minio**) are generally convenient options for object storage. + +In this tutorial, we will run a `minio-server` as object storage in our local `kind` cluster using `minio-operator` and create a bucket named `milvus` in it, which the deployed milvus database will use. + +```bash + +$ helm repo add minio https://operator.min.io/ +$ helm repo update minio +$ helm upgrade --install --namespace "minio-operator" --create-namespace "minio-operator" minio/operator --set operator.replicaCount=1 + +$ helm upgrade --install --namespace "demo" --create-namespace milvus-minio minio/tenant \ +--set tenant.pools[0].servers=1 \ +--set tenant.pools[0].volumesPerServer=1 \ +--set tenant.pools[0].size=1Gi \ +--set tenant.certificate.requestAutoCert=false \ +--set tenant.buckets[0].name="milvus" \ +--set tenant.pools[0].name="default" + +``` + +Now we need to create a `Secret` named `my-release-minio`. It contains the necessary connection information using which the milvus database will connect to the object storage. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-release-minio + namespace: demo +stringData: + milvus.storage.type: "s3" + milvus.storage.bucket: "milvus" + milvus.storage.baseKey: "milvus/segments" + milvus.s3.accessKey: "minio" + milvus.s3.secretKey: "minio123" + milvus.s3.protocol: "http" + milvus.s3.enablePathStyleAccess: "true" + milvus.s3.endpoint.signingRegion: "us-east-1" + milvus.s3.endpoint.url: "http://myminio-hl.demo.svc.cluster.local:9000/" +``` + +Let’s create the `deep-storage-config` Secret shown above: + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/deep-storage-config.yaml +secret/deep-storage-config created +``` + +You can also use options like **Amazon S3**, **Google Cloud Storage**, **Azure Blob Storage** or **HDFS** and create a connection information `Secret` like this, and you are good to go. + +## Create a Milvus Database + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Milvus +metadata: + name: milvus-cluster + namespace: demo +spec: + version: "2.6.11" + objectStorage: + configSecret: + name: "my-release-minio" + topology: + mode: Distributed + distributed: + mixcoord: + replicas: 2 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + storageClassName: local-path + resources: + requests: + storage: 10Gi +``` + +```bash +kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/milvus/quickstart/distributed.yaml +kubectl get milvus -n demo milvus-cluster -w +``` + +## Verify Milvus Database + +```bash +kubectl get milvus -n demo +kubectl describe milvus -n demo milvus-cluster +``` + +When `status.phase` becomes `Ready`, the Milvus deployment is ready to serve vector search traffic. + +## Cleaning up + +```bash +kubectl delete milvus -n demo milvus-cluster +kubectl delete ns demo +``` \ No newline at end of file diff --git a/docs/guides/milvus/tls/_index.md b/docs/guides/milvus/tls/_index.md new file mode 100644 index 000000000..0a6bca5d2 --- /dev/null +++ b/docs/guides/milvus/tls/_index.md @@ -0,0 +1,10 @@ +--- +title: Milvus TLS +menu: + docs_{{ .version }}: + identifier: milvus-tls + name: TLS + parent: milvus-guides + weight: 25 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/milvus/tls/overview.md b/docs/guides/milvus/tls/overview.md new file mode 100644 index 000000000..979dff780 --- /dev/null +++ b/docs/guides/milvus/tls/overview.md @@ -0,0 +1,39 @@ +--- +title: Milvus TLS Overview +menu: + docs_{{ .version }}: + identifier: milvus-tls-overview + name: Overview + parent: milvus-tls + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Milvus TLS + +This guide shows the main TLS considerations for a Milvus deployment. + +## Before You Begin + +- Install `cert-manager` in your cluster. +- Deploy Milvus first using the [quickstart guide](/docs/guides/milvus/quickstart/quickstart.md). + +## Configure TLS + +Milvus TLS setup is managed with cert-manager and KubeDB TLS settings. + +- Use a trusted issuer for server certificates. +- Rotate certificates in a controlled maintenance window. + +## Verify + +```bash +kubectl get milvus -n demo milvus-cluster -o yaml +kubectl get secret -n demo +``` + +## Next Steps + +- Test client connectivity with TLS enabled before routing production traffic. +- Document the certificate rotation process for your team.