Add Helm job for OpenSearch index creation #36
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| { | ||
| "mappings": { | ||
| "properties": { | ||
| "object_ref": { "type": "keyword" }, | ||
|
Comment on lines
+1
to
+4
Contributor
Author
There was a problem hiding this comment. Mappings-only is intentional — OpenSearch defaults for shards/replicas are appropriate. The PR description referencing settings was outdated (old index definition replaced in commit 8971941). |
||
| "object_type": { "type": "keyword" }, | ||
| "object_id": { "type": "keyword" }, | ||
| "parent_refs": { "type": "keyword" }, | ||
| "sort_name": { "type": "keyword" }, | ||
| "name_and_aliases": { "type": "search_as_you_type" }, | ||
| "tags": { "type": "keyword" }, | ||
| "public": { "type": "boolean" }, | ||
| "access_check_query": { | ||
| "type": "text", | ||
| "fields": { | ||
| "keyword": { "type": "keyword", "ignore_above": 256 } | ||
| } | ||
| }, | ||
| "access_check_object": { "type": "keyword" }, | ||
| "access_check_relation": { "type": "keyword" }, | ||
| "history_check_query": { | ||
| "type": "text", | ||
| "fields": { | ||
| "keyword": { "type": "keyword", "ignore_above": 256 } | ||
| } | ||
| }, | ||
| "history_check_object": { "type": "keyword" }, | ||
| "history_check_relation": { "type": "keyword" }, | ||
| "latest": { "type": "boolean" }, | ||
| "created_at": { "type": "date" }, | ||
| "created_by": { "type": "keyword" }, | ||
| "created_by_principals": { "type": "keyword" }, | ||
| "created_by_emails": { "type": "keyword" }, | ||
| "updated_at": { "type": "date" }, | ||
| "updated_by": { "type": "keyword" }, | ||
| "updated_by_principals": { "type": "keyword" }, | ||
| "updated_by_emails": { "type": "keyword" }, | ||
| "deleted_at": { "type": "date" }, | ||
| "deleted_by": { "type": "keyword" }, | ||
| "deleted_by_principals": { "type": "keyword" }, | ||
| "deleted_by_emails": { "type": "keyword" }, | ||
| "scheduled_start_time": { "type": "date" }, | ||
| "scheduled_end_time": { "type": "date" }, | ||
| "data": { | ||
| "type": "flat_object" | ||
| }, | ||
| "fulltext": { | ||
| "type": "match_only_text" | ||
| }, | ||
| "contacts": { | ||
| "type": "nested", | ||
| "properties": { | ||
| "lfx_principal": { "type": "search_as_you_type" }, | ||
| "name": { "type": "search_as_you_type" }, | ||
| "emails": { "type": "search_as_you_type" }, | ||
| "bot": { "type": "boolean" }, | ||
| "profile": { "type": "flat_object" } | ||
| } | ||
| }, | ||
| "v1_data": { | ||
| "type": "flat_object" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| # Copyright The Linux Foundation and each contributor to LFX. | ||
| # SPDX-License-Identifier: MIT | ||
| {{- if .Values.opensearch.indexingJob.enabled }} | ||
| --- | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: {{ .Release.Name }}-opensearch-index-config | ||
| namespace: {{ .Release.Namespace }} | ||
|
Comment on lines
+8
to
+9
Contributor
Author
There was a problem hiding this comment. The Job and ConfigMap use
Comment on lines
+8
to
+9
Contributor
Author
There was a problem hiding this comment. The Job and ConfigMap use |
||
| annotations: | ||
| "helm.sh/hook": post-install,post-upgrade | ||
| "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded | ||
|
|
||
| "helm.sh/hook-weight": "0" | ||
| data: | ||
| resources-index.json: | | ||
|
Contributor
There was a problem hiding this comment. This index json content doesn't quite match what we should be actually using, which is https://github.com/linuxfoundation/lfx-architecture-scratch/blob/main/2024-12%20ReBAC%20Demo/opensearch-resources-index.json. But on that note, could we have this json content in a file instead and then have it read into the k8 template? I think that would be better.
Contributor
Author
There was a problem hiding this comment. Thanks for the feedback! I realized I hadn't pulled the repo for a while and was using an outdated index. I've moved the index JSON to it's own file and have it being included now with
Contributor
Author
There was a problem hiding this comment. Both concerns are fully addressed — the index JSON was updated (commit 8971941) to match the correct schema and moved to |
||
| {{- .Files.Get "files/opensearch-resources-index.json" | nindent 4 }} | ||
| {{- end }} | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| # Copyright The Linux Foundation and each contributor to LFX. | ||
| # SPDX-License-Identifier: MIT | ||
| {{- if .Values.opensearch.indexingJob.enabled }} | ||
| {{- $job := .Values.opensearch.indexingJob }} | ||
| --- | ||
| apiVersion: batch/v1 | ||
| kind: Job | ||
| metadata: | ||
| name: {{ .Release.Name }}-opensearch-index-setup | ||
| namespace: {{ .Release.Namespace }} | ||
| annotations: | ||
| "helm.sh/hook": post-install,post-upgrade | ||
| "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded | ||
| "helm.sh/hook-weight": "1" | ||
| spec: | ||
| ttlSecondsAfterFinished: {{ $job.ttlSecondsAfterFinished }} | ||
| backoffLimit: {{ $job.backoffLimit }} | ||
| {{- if $job.activeDeadlineSeconds }} | ||
| activeDeadlineSeconds: {{ $job.activeDeadlineSeconds }} | ||
| {{- end }} | ||
|
Comment on lines
+6
to
+20
Contributor
Author
There was a problem hiding this comment. Fixed in commit 1a85468 — converted both the Job and ConfigMap to Helm hooks ( |
||
| template: | ||
| spec: | ||
| restartPolicy: {{ $job.restartPolicy }} | ||
| volumes: | ||
| - name: index-config | ||
| configMap: | ||
| name: {{ .Release.Name }}-opensearch-index-config | ||
| containers: | ||
| - name: curl | ||
| image: {{ $job.image.repository }}:{{ $job.image.tag }} | ||
| imagePullPolicy: {{ $job.image.pullPolicy }} | ||
| {{- with $job.resources }} | ||
| resources: | ||
| {{- toYaml . | nindent 12 }} | ||
| {{- end }} | ||
| {{- if .Values.opensearch.auth.enabled }} | ||
| env: | ||
| - name: OPENSEARCH_USERNAME | ||
| {{- if .Values.opensearch.auth.existingSecret }} | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: {{ .Values.opensearch.auth.existingSecret }} | ||
| key: username | ||
| {{- else }} | ||
| value: {{ .Values.opensearch.auth.username | quote }} | ||
| {{- end }} | ||
| - name: OPENSEARCH_PASSWORD | ||
| {{- if .Values.opensearch.auth.existingSecret }} | ||
| valueFrom: | ||
| secretKeyRef: | ||
| name: {{ .Values.opensearch.auth.existingSecret }} | ||
| key: password | ||
| {{- else }} | ||
| value: {{ .Values.opensearch.auth.password | quote }} | ||
| {{- end }} | ||
| {{- end }} | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| volumeMounts: | ||
| - name: index-config | ||
| mountPath: /config | ||
| readOnly: true | ||
|
Comment on lines
+29
to
+62
There was a problem hiding this comment. The Job container doesn't define a securityContext, while the main application deployment includes one with
Contributor
Author
There was a problem hiding this comment. Fixed in commit 1a85468 — added |
||
| command: | ||
| - sh | ||
| - -c | ||
| - | | ||
| set -e | ||
| OPENSEARCH_URL="{{ .Values.opensearch.url | trimSuffix "/" }}" | ||
| INDEX_NAME="{{ .Values.opensearch.index }}" | ||
|
|
||
| # Check if index already exists | ||
| HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" {{ if .Values.opensearch.auth.enabled }}-u "${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}" {{ end }}--head "${OPENSEARCH_URL}/${INDEX_NAME}") | ||
|
|
||
| if [ "$HTTP_CODE" = "200" ]; then | ||
| echo "Index '${INDEX_NAME}' already exists, skipping creation" | ||
| exit 0 | ||
| elif [ "$HTTP_CODE" = "404" ]; then | ||
| echo "Index '${INDEX_NAME}' does not exist, creating..." | ||
| curl -f {{ if .Values.opensearch.auth.enabled }}-u "${OPENSEARCH_USERNAME}:${OPENSEARCH_PASSWORD}" {{ end }}-X PUT "${OPENSEARCH_URL}/${INDEX_NAME}" \ | ||
| -H 'Content-Type: application/json' \ | ||
| -d @/config/resources-index.json | ||
| echo "Index '${INDEX_NAME}' created successfully" | ||
| else | ||
| echo "Unexpected response checking index: HTTP ${HTTP_CODE}" | ||
| exit 1 | ||
| fi | ||
|
Comment on lines
+71
to
+86
There was a problem hiding this comment. 🧩 Analysis chain🏁 Script executed: cd charts/lfx-v2-indexer-service && find . -type f -name "*.yaml" -o -name "*.yml" | head -20Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 193 🏁 Script executed: cat -n charts/lfx-v2-indexer-service/templates/job.yaml | sed -n '60,90p'Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 1524 🏁 Script executed: cat charts/lfx-v2-indexer-service/values.yamlRepository: linuxfoundation/lfx-v2-indexer-service Length of output: 4563 🏁 Script executed: #!/bin/bash
# Check if the image tag exists on Docker Hub
echo "Checking Docker Hub for curlimages/curl:8.11.1..."
DOCKER_RESPONSE=$(curl -s "https://hub.docker.com/v2/repositories/curlimages/curl/tags/8.11.1/")
echo "$DOCKER_RESPONSE" | jq -r '.name // "NOT_FOUND"'
# Also check available tags to verify repository access
echo -e "\nRecent available tags (first 5):"
curl -s "https://hub.docker.com/v2/repositories/curlimages/curl/tags/?page_size=5" | jq -r '.results[].name'Repository: linuxfoundation/lfx-v2-indexer-service Length of output: 276 🌐 Web query:
💡 Result: curl 8.11.1 had multiple security advisories fixed in curl 8.12.0 (released Feb 5, 2025). Summary of CVEs affecting 8.11.1:
Recommended action: upgrade curl/libcurl to 8.12.0 (or later) or apply the vendor patches; if you cannot upgrade immediately follow the per-advisory mitigations (avoid netrc+redirects, avoid CURLOPT_ACCEPT_ENCODING with ancient zlib, disable eventfd) until patched. [1][2][3] Sources: Upgrade curl image to version 8.12.0 or later to address known security advisories. The index creation logic is well-structured with proper idempotency checks (HEAD before PUT) and appropriate error handling for different HTTP response codes. However, curlimages/curl:8.11.1 has three low-severity security advisories that were fixed in 8.12.0 (released February 5, 2025):
Update the image tag in values.yaml from 🤖 Prompt for AI Agents
Contributor
Author
There was a problem hiding this comment. The image version is configurable via There was a problem hiding this comment.
Would you like me to open a GitHub issue to track bumping the default
Comment on lines
+71
to
+86
Contributor
Author
There was a problem hiding this comment. The Kubernetes Job |
||
| {{- end }} | ||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -25,6 +25,36 @@ opensearch: | |||||||||||||||||||||||||
| url: http://opensearch-cluster-master.lfx.svc.cluster.local:9200/ | ||||||||||||||||||||||||||
| # index is the index name for storing resources | ||||||||||||||||||||||||||
| index: resources | ||||||||||||||||||||||||||
| # auth configures authentication for OpenSearch | ||||||||||||||||||||||||||
| auth: | ||||||||||||||||||||||||||
| # enabled controls whether authentication is used | ||||||||||||||||||||||||||
| enabled: false | ||||||||||||||||||||||||||
| # existingSecret is the name of an existing secret containing credentials | ||||||||||||||||||||||||||
| # The secret should have 'username' and 'password' keys | ||||||||||||||||||||||||||
| existingSecret: "" | ||||||||||||||||||||||||||
| # username is the OpenSearch username (ignored if existingSecret is set) | ||||||||||||||||||||||||||
| username: "" | ||||||||||||||||||||||||||
| # password is the OpenSearch password (ignored if existingSecret is set) | ||||||||||||||||||||||||||
| password: "" | ||||||||||||||||||||||||||
|
Comment on lines
+35
to
+38
There was a problem hiding this comment. Avoid inline OpenSearch credentials in chart values. Line 35-Line 38 introduce plaintext Suggested hardening diff auth:
# enabled controls whether authentication is used
enabled: false
# existingSecret is the name of an existing secret containing credentials
# The secret should have 'username' and 'password' keys
existingSecret: ""
- # username is the OpenSearch username (ignored if existingSecret is set)
- username: ""
- # password is the OpenSearch password (ignored if existingSecret is set)
- password: ""
+ # WARNING: avoid inline credentials in values files (GitOps/VCS leak risk).
+ # Prefer `existingSecret` and leave inline credentials unset.
+ username: null
+ password: null📝 Committable suggestion
Suggested change
🤖 Prompt for AI Agents
Contributor
Author
There was a problem hiding this comment. The There was a problem hiding this comment.
✏️ Learnings added
Comment on lines
+28
to
+38
Contributor
Author
There was a problem hiding this comment.
Comment on lines
+28
to
+38
Contributor
Author
There was a problem hiding this comment.
|
||||||||||||||||||||||||||
| # indexingJob is the configuration for the OpenSearch index creation job | ||||||||||||||||||||||||||
| indexingJob: | ||||||||||||||||||||||||||
| # enabled is a boolean to determine if the indexing job should be created | ||||||||||||||||||||||||||
| enabled: true | ||||||||||||||||||||||||||
|
coderabbitai[bot] marked this conversation as resolved.
Comment on lines
+41
to
+42
|
||||||||||||||||||||||||||
| # backoffLimit is the number of retries before marking the job as failed | ||||||||||||||||||||||||||
|
Comment on lines
+39
to
+43
Contributor
Author
There was a problem hiding this comment. The chart version is dynamically replaced during the CI build — |
||||||||||||||||||||||||||
| backoffLimit: 3 | ||||||||||||||||||||||||||
| # ttlSecondsAfterFinished is how long to keep the job after completion | ||||||||||||||||||||||||||
|
Comment on lines
+39
to
+45
Contributor
Author
There was a problem hiding this comment. Chart version is dynamically replaced by the CI build — |
||||||||||||||||||||||||||
| ttlSecondsAfterFinished: 300 | ||||||||||||||||||||||||||
| # activeDeadlineSeconds is the maximum time for the job to run (optional) | ||||||||||||||||||||||||||
| activeDeadlineSeconds: null | ||||||||||||||||||||||||||
| # restartPolicy is the pod restart policy (OnFailure or Never) | ||||||||||||||||||||||||||
| restartPolicy: OnFailure | ||||||||||||||||||||||||||
| # image is the container image for the job | ||||||||||||||||||||||||||
| image: | ||||||||||||||||||||||||||
| repository: curlimages/curl | ||||||||||||||||||||||||||
| tag: "8.11.1" | ||||||||||||||||||||||||||
| pullPolicy: IfNotPresent | ||||||||||||||||||||||||||
| # resources defines CPU and memory limits/requests for the job container | ||||||||||||||||||||||||||
| resources: {} | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
| # heimdall is the configuration for the heimdall middleware | ||||||||||||||||||||||||||
| heimdall: | ||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Mappings-only is intentional — OpenSearch's default shard/replica counts are appropriate for our deployment and we don't need to manage them via Helm values at this stage. The PR description was outdated; it previously referenced settings from an old index definition that was replaced in commit 8971941.