From 23fd631afbc4d4bf0944535ae56950a0cdc1aa23 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 29 Apr 2026 13:51:10 +0200 Subject: [PATCH 1/6] Add yProvStore template --- artifacts/yprovstore.yml | 134 +++++++++++++++++++++++++++++++++ templates/simple-node-disk.yml | 1 + templates/yprovstore.yaml | 99 ++++++++++++++++++++++++ 3 files changed, 234 insertions(+) create mode 100644 artifacts/yprovstore.yml create mode 100644 templates/yprovstore.yaml diff --git a/artifacts/yprovstore.yml b/artifacts/yprovstore.yml new file mode 100644 index 00000000..6656a42e --- /dev/null +++ b/artifacts/yprovstore.yml @@ -0,0 +1,134 @@ +--- +- name: Install yProvStore + hosts: localhost + connection: local + vars: + branch: "v2.0.1" + traefik_letsencrypt_email: "{{ letsencrypt_email | default('server@yprovstore.fedcloud.eu') }}" + traefik_dns_name: "{{ dns_name | default('yprovstore.fedcloud.eu') }}" + traefik_public_ip: "{{ public_ip | default(ansible_default_ipv4.address) }}" + minio_endpoint: "localhost:9000" + minio_root_password: "minio123" + minio_root_user: "minio" + minio_bucket_name: "yprov-documents" + minio_secure: "False" + roles: + - role: 'grycap.docker' + tasks: + - name: check if yProvStore files are already downloaded + stat: + path: /opt/yProvStore/docker-compose.yml + register: docker_file_stat + + - name: Download yProvStore files + git: + repo: 'https://github.com/HPCI-Lab/yProvStore' + dest: /opt/yProvStore + version: "{{ branch }}" + when: not docker_file_stat.stat.exists + + - name: copy .env.example to .env + copy: + src: /opt/yProvStore/.env.example + dest: /opt/yProvStore/.env + remote_src: yes + mode: '644' + force: false + + - name: Create keys directory + file: + path: /opt/yProvStore/keys/ + state: directory + mode: '644' + + - name: Create private key file for + copy: + content: "" + dest: /opt/yProvStore/keys/user_private.pem + mode: '644' + force: false + + - name: Set MinIO connection variables + set_fact: + minio_endpoint: "{{ storage_connection_string | urlsplit('hostname') }}" + minio_root_password: "{{ storage_connection_string | urlsplit('password') }}" + minio_root_user: "{{ storage_connection_string | urlsplit('username') }}" + minio_bucket_name: "{{ storage_connection_string | urlsplit('path') | regex_replace('^/', '') }}" + minio_secure: "True" + when: storage_connection_string is defined and storage_connection_string != '' + + - name: Update .env file + lineinfile: + path: /opt/yProvStore/.env + regexp: '^{{ item.var }}=' + line: '{{ item.var }}={{ item.value }}' + loop: + - { var: 'USE_LOCAL_PID_SERVICE', value: 'True' } + - { var: 'MINIO_ROOT_USER', value: '{{ minio_root_user }}' } + - { var: 'MINIO_ROOT_PASSWORD', value: '{{ minio_root_password }}' } + - { var: 'MINIO_BUCKET_NAME', value: '{{ minio_bucket_name }}' } + - { var: 'MINIO_ENDPOINT', value: '{{ minio_endpoint }}' } + - { var: 'MINIO_SECURE', value: '{{ minio_secure }}' } + - { var: 'POSTGRES_USER', value: 'postgres' } + - { var: 'POSTGRES_PASSWORD', value: 'postgres123' } + - { var: 'POSTGRES_DB', value: 'yprovstore' } + - { var: 'APP_PORT', value: '8000' } + - { var: 'DOCUMENT_SIZE_LIMIT', value: '{{ document_size_limit | default(100) }}' } + + - name: Update .env file + lineinfile: + path: /opt/yProvStore/.env + regexp: '^DB_CONNECTION_STRING=' + line: 'DB_CONNECTION_STRING={{ db_connection_string }}' + when: db_connection_string is defined and db_connection_string != '' + + - name: Update docker-compose.yml with Traefik configuration + blockinfile: + path: /opt/yProvStore/docker-compose.yml + insertbefore: '^ minio:' + prepend_newline: true + block: | + # Traefik reverse proxy configuration + labels: + - "traefik.enable=true" + - "traefik.http.routers.api.service=api" + - "traefik.http.routers.api.rule=Host(`{{ traefik_dns_name }}`, `{{ traefik_public_ip }}`)" + - "traefik.http.routers.api.entrypoints=websecure" + - "traefik.http.routers.api.tls.certresolver=myresolver" + - "traefik.http.services.api.loadbalancer.server.port=8000" + traefik: + image: "traefik:v2.11" + container_name: "traefik" + command: + # - "--log.level=DEBUG" + - "--api.insecure=true" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.websecure.address=:443" + - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + - "--certificatesresolvers.myresolver.acme.httpchallenge=true" + - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" + #- "--certificatesresolvers.myresolver.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" + - "--certificatesresolvers.myresolver.acme.email={{ traefik_letsencrypt_email }}" + - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" + ports: + - "80:80" + - "443:443" + - "8080:8080" + volumes: + - "./letsencrypt:/letsencrypt" + - "/var/run/docker.sock:/var/run/docker.sock:ro" + + - name: Add curl package in Dockerfile + lineinfile: + path: /opt/yProvStore/Dockerfile + line: ' curl \' + insertafter: ' dos2unix \\' + + - name: Exec docker-compose up + docker_compose: + project_src: /opt/yProvStore/ + state: present diff --git a/templates/simple-node-disk.yml b/templates/simple-node-disk.yml index 347837df..7cc11d2e 100644 --- a/templates/simple-node-disk.yml +++ b/templates/simple-node-disk.yml @@ -60,6 +60,7 @@ metadata: - interoperability-registry-read.yml - pyhandle.yml - apptainer.yaml + - yprovstore.yaml topology_template: diff --git a/templates/yprovstore.yaml b/templates/yprovstore.yaml new file mode 100644 index 00000000..576771ae --- /dev/null +++ b/templates/yprovstore.yaml @@ -0,0 +1,99 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +imports: + - grycap_custom_types: https://raw.githubusercontent.com/grycap/tosca/main/custom_types.yaml + + +description: Deploy yProvStore, a provenance store for scientific workflows. + +metadata: + template_version: "1.0.0" + template_name: yProvStore + display_name: Deploy yProvStore + web: https://github.com/HPCI-Lab/yProvStore + parents: + - simple-node-disk.yml + tabs: + yProvStore: + - letsencrypt_email + - db_connection_string + - storage_connection_string + - document_size_limit + +topology_template: + inputs: + + letsencrypt_email: + default: changeme@email.com + description: Email to be used for Let's Encrypt certificates + type: string + + db_connection_string: + default: '' + description: Database connection string for yProvStore (format postgresql://user:pass@pgserver/db) + type: string + storage_connection_string: + default: '' + description: Storage connection string for yProvStore (format s3://access_key:secret_key@endpoint/bucket) + type: string + document_size_limit: + default: 100 + description: Maximum size in MB of documents that can be stored in yProvStore + type: integer + + # Restrict some simple-node-disk input values + + storage_size: + type: scalar-unit.size + description: Size of the extra HD added to the instance (Set 0 if disk is not needed) + default: 50 GiB + constraints: + - valid_values: [ 50 GiB, 100 GiB, 200 GiB, 500 GiB, 1 TiB, 2 TiB, 10 TiB, 20 TiB, 40 TiB, 100 TiB ] + + mount_path: + type: string + description: Path to mount the extra disk + default: /data + + dns_name: + default: yprovstore-test.vm.fedcloud.eu + description: DNS hostname to access yProvStore + type: string + + node_templates: + + yprovstore: + type: tosca.nodes.ec3.Application + artifacts: + docker_role: + file: grycap.docker + type: tosca.artifacts.AnsibleGalaxy.role + capabilities: + endpoint: + properties: + ports: + port_http: + protocol: tcp + source: 80 + port_https: + protocol: tcp + source: 443 + interfaces: + Standard: + configure: + implementation: https://raw.githubusercontent.com/grycap/tosca/eosc_beyond/artifacts/yprovstore.yml + inputs: + letsencrypt_email: { get_input: letsencrypt_email } + dns_name: { get_input: dns_name } + public_ip: { get_attribute: [ simple_node, public_address, 0 ] } + db_connection_string: { get_input: db_connection_string } + storage_connection_string: { get_input: storage_connection_string } + document_size_limit: { get_input: document_size_limit } + requirements: + - host: simple_node + + outputs: + yprovstore_url: + value: { concat: [ 'https://', get_attribute: [ simple_node, public_address, 0 ], '/docs' ] } + yprovstore_url_dns: + value: { concat: [ 'https://', get_input: [ dns_name ], '/docs' ] } \ No newline at end of file From c18b85f6b6f0c1306932b098cf713b95cd8a9a0d Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 29 Apr 2026 13:56:23 +0200 Subject: [PATCH 2/6] Fix style --- templates/yprovstore.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/yprovstore.yaml b/templates/yprovstore.yaml index 576771ae..8e81ac26 100644 --- a/templates/yprovstore.yaml +++ b/templates/yprovstore.yaml @@ -96,4 +96,4 @@ topology_template: yprovstore_url: value: { concat: [ 'https://', get_attribute: [ simple_node, public_address, 0 ], '/docs' ] } yprovstore_url_dns: - value: { concat: [ 'https://', get_input: [ dns_name ], '/docs' ] } \ No newline at end of file + value: { concat: [ 'https://', get_input: [ dns_name ], '/docs' ] } From d34a91ee98046bf127691cfe64a541bd9d4f008a Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Wed, 29 Apr 2026 13:57:37 +0200 Subject: [PATCH 3/6] Fix style --- artifacts/yprovstore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/artifacts/yprovstore.yml b/artifacts/yprovstore.yml index 6656a42e..eb579b47 100644 --- a/artifacts/yprovstore.yml +++ b/artifacts/yprovstore.yml @@ -31,7 +31,7 @@ copy: src: /opt/yProvStore/.env.example dest: /opt/yProvStore/.env - remote_src: yes + remote_src: true mode: '644' force: false From b397638500ce45c299127c41aa6400a4a4b31a8f Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 30 Apr 2026 12:24:38 +0200 Subject: [PATCH 4/6] Improve text message --- templates/yprovstore.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templates/yprovstore.yaml b/templates/yprovstore.yaml index 8e81ac26..ea6e3b32 100644 --- a/templates/yprovstore.yaml +++ b/templates/yprovstore.yaml @@ -30,11 +30,15 @@ topology_template: db_connection_string: default: '' - description: Database connection string for yProvStore (format postgresql://user:pass@pgserver/db) + description: | + Database connection string for yProvStore (format postgresql://user:pass@pgserver/db). + If not provided, a local PostgreSQL instance will be deployed and used. type: string storage_connection_string: default: '' - description: Storage connection string for yProvStore (format s3://access_key:secret_key@endpoint/bucket) + description: | + Storage connection string for yProvStore (format s3://access_key:secret_key@endpoint/bucket). + If not provided, a local MinIO instance will be deployed and used. type: string document_size_limit: default: 100 From 94b127b99db2c66b94243c365b9a1533b8be7c07 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 7 May 2026 12:16:50 +0200 Subject: [PATCH 5/6] Add OIDC fields --- artifacts/yprovstore.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/artifacts/yprovstore.yml b/artifacts/yprovstore.yml index eb579b47..664f9642 100644 --- a/artifacts/yprovstore.yml +++ b/artifacts/yprovstore.yml @@ -82,6 +82,19 @@ line: 'DB_CONNECTION_STRING={{ db_connection_string }}' when: db_connection_string is defined and db_connection_string != '' + - name: Update .env file with OIDC authentication variables + lineinfile: + path: /opt/yProvStore/.env + regexp: '^{{ item.var }}=' + line: '{{ item.var }}={{ item.value }}' + loop: + - { var: 'USE_EGI_CHECKIN_AUTH', value: 'True' } + - { var: 'EGI_CHECKIN_INTROSPECTION_ENDPOINT', value: '{{ oidc_instrospection_endpoint }}' } + - { var: 'EGI_CHECKIN_CLIENT_ID', value: '{{ oidc_client_id }}' } + - { var: 'EGI_CHECKIN_CLIENT_SECRET', value: '{{ oidc_client_secret }}' } + - { var: 'EGI_CHECKIN_REQUIRED_ENTITLEMENTS', value: '{{ oidc_required_entitlements }}' } + when: use_oidc_auth | bool | default(false) + - name: Update docker-compose.yml with Traefik configuration blockinfile: path: /opt/yProvStore/docker-compose.yml From fbb734fb5b7183f7ba07d274e72bd44fda59b915 Mon Sep 17 00:00:00 2001 From: Miguel Caballer Date: Thu, 7 May 2026 12:25:28 +0200 Subject: [PATCH 6/6] Add OIDC fields --- templates/yprovstore.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/templates/yprovstore.yaml b/templates/yprovstore.yaml index ea6e3b32..2e2daaae 100644 --- a/templates/yprovstore.yaml +++ b/templates/yprovstore.yaml @@ -19,6 +19,15 @@ metadata: - db_connection_string - storage_connection_string - document_size_limit + - use_oidc_auth + - oidc_instrospection_endpoint: + enabled_by: use_oidc_auth + - oidc_required_entitlements: + enabled_by: use_oidc_auth + - oidc_client_id: + enabled_by: use_oidc_auth + - oidc_client_secret: + enabled_by: use_oidc_auth topology_template: inputs: @@ -59,6 +68,27 @@ topology_template: description: Path to mount the extra disk default: /data + use_oidc_auth: + type: boolean + description: Whether to use OIDC authentication for yProvStore + default: false + oidc_instrospection_endpoint: + type: string + description: OIDC introspection endpoint + default: 'https://enes-proxy.pilot.eosc-beyond.eu/auth/realms/enes/protocol/openid-connect/token/introspect' + oidc_required_entitlements: + type: string + description: Required entitlement to access yProvStore + default: 'urn:mace:egi.eu:group:enes.pilot.eosc-beyond.eu:role=member#aai.egi.eu' + oidc_client_id: + type: string + description: OIDC Client ID + default: '' + oidc_client_secret: + type: string + description: OIDC Client Secret + default: '' + dns_name: default: yprovstore-test.vm.fedcloud.eu description: DNS hostname to access yProvStore @@ -93,6 +123,12 @@ topology_template: db_connection_string: { get_input: db_connection_string } storage_connection_string: { get_input: storage_connection_string } document_size_limit: { get_input: document_size_limit } + use_oidc_auth: { get_input: use_oidc_auth } + oidc_instrospection_endpoint: { get_input: oidc_instrospection_endpoint } + oidc_required_entitlements: { get_input: oidc_required_entitlements } + oidc_client_id: { get_input: oidc_client_id } + oidc_client_secret: { get_input: oidc_client_secret } + requirements: - host: simple_node