feat: add YouTube knowledge source ingestion and retrieval

.github/workflows/chart-lint-publish.yml

@@ -0,0 +1,59 @@
+
+name: Validate / Publish helm charts
+
+on:
+  release:
+    types: [published]
+  pull_request:
+    types: [opened, reopened, synchronize]
+    paths:
+      - 'github-activity-tracker/helm/**'
+  workflow_dispatch:
+    inputs:
+      IGNORE_CHARTS:
+        description: 'Provide list of charts to be ignored separated by pipe(|)'
+        required: false
+        default: ''
+        type: string
+      CHART_PUBLISH:
+        description: 'Chart publishing to gh-pages branch'
+        required: false
+        default: 'NO'
+        type: string
+        options:
+          - YES
+          - NO
+      INCLUDE_ALL_CHARTS:
+        description: 'Include all charts for Linting/Publishing (YES/NO)'
+        required: false
+        default: 'NO'
+        type: string
+        options:
+          - YES
+          - NO
+  push:
+    branches:
+      - '!release-branch'
+      - master
+      - develop
+    paths:
+      - 'github-activity-tracker/helm/**'
+
+jobs:
+  chart-lint-publish:
+    uses: mosip/kattu/.github/workflows/chart-lint-publish.yml@master
+    with:
+      CHARTS_DIR: ./github-activity-tracker/helm
+      CHARTS_URL: https://mosip.github.io/mosip-helm
+      REPOSITORY: mosip-helm
+      BRANCH: gh-pages
+      INCLUDE_ALL_CHARTS: "${{ inputs.INCLUDE_ALL_CHARTS || 'NO' }}"
+      IGNORE_CHARTS: "${{ inputs.IGNORE_CHARTS ||'redis' }}"
+      CHART_PUBLISH: "${{ inputs.CHART_PUBLISH || 'YES' }}"
+      LINTING_CHART_SCHEMA_YAML_URL: "https://raw.githubusercontent.com/mosip/kattu/master/.github/helm-lint-configs/chart-schema.yaml"
+      LINTING_LINTCONF_YAML_URL: "https://raw.githubusercontent.com/mosip/kattu/master/.github/helm-lint-configs/lintconf.yaml"
+      LINTING_CHART_TESTING_CONFIG_YAML_URL: "https://raw.githubusercontent.com/mosip/kattu/master/.github/helm-lint-configs/chart-testing-config.yaml"
+      LINTING_HEALTH_CHECK_SCHEMA_YAML_URL: "https://raw.githubusercontent.com/mosip/kattu/master/.github/helm-lint-configs/health-check-schema.yaml"
+    secrets:
+      TOKEN: ${{ secrets.ACTION_PAT }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

.github/workflows/mosip-nexus-docker-publish.yml

@@ -0,0 +1,45 @@
+name: MosipNexus — Build and Publish Docker Image
+
+on:
+  push:
+    branches:
+      - master
+      - develop
+    paths:
+      - "MosipNexus/**"
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Compute image tags
+        id: tags
+        run: |
+          SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
+          if [ "${{ github.ref_name }}" = "master" ]; then
+            echo "tags=${{ secrets.DOCKERHUB_USERNAME }}/mosip-nexus:latest,${{ secrets.DOCKERHUB_USERNAME }}/mosip-nexus:${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          else
+            echo "tags=${{ secrets.DOCKERHUB_USERNAME }}/mosip-nexus:develop,${{ secrets.DOCKERHUB_USERNAME }}/mosip-nexus:develop-${SHORT_SHA}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: ./MosipNexus
+          push: true
+          tags: ${{ steps.tags.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/push-trigger.yml

@@ -0,0 +1,91 @@
+name: Maven Package upon a push
+
+on:
+  release:
+    types: [published]
+  pull_request:
+    types: [opened, reopened, synchronize]
+  workflow_dispatch:
+    inputs:
+      message:
+        description: 'Message for manually triggering'
+        required: false
+        default: 'Triggered for Updates'
+        type: string
+  push:
+    branches:
+      - '!release-branch'
+      - master
+      - 1.*
+      - develop
+      - MOSIP*
+      - release*
+
+jobs:
+  build-github-activity-tracker-service:
+    uses: mosip/kattu/.github/workflows/npm-build.yml@develop
+    with:
+      SERVICE_LOCATION: ./github-activity-tracker/backend
+      BUILD_ARTIFACT: github-activity-tracker-service
+      NPM_BUILD_TYPE: 'BOB'
+      NODE_VERSION: "18.x"
+      ZIP_DIR: "dist"
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+
+  build-github-activity-tracker-ui:
+    uses: mosip/kattu/.github/workflows/npm-build.yml@develop
+    with:
+      SERVICE_LOCATION: ./github-activity-tracker/frontend
+      BUILD_ARTIFACT: github-activity-tracker-ui
+      NPM_BUILD_TYPE: 'BOB'
+      NODE_VERSION: "18.x"
+      ZIP_DIR: "dist"
+    secrets:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+
+  build-dockers-github-activity-tracker-ui:
+    needs: build-github-activity-tracker-ui
+    strategy:
+      matrix:
+        include:
+          - SERVICE_LOCATION: ./github-activity-tracker/frontend
+            SERVICE_NAME: 'github-activity-tracker-ui'
+            NPM_BUILD: true
+            BUILD_ARTIFACT: 'github-activity-tracker-ui'
+      fail-fast: false
+    name: ${{ matrix.SERVICE_NAME }}
+    uses: mosip/kattu/.github/workflows/docker-build.yml@master-java21
+    with:
+      SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }}
+      SERVICE_NAME: ${{ matrix.SERVICE_NAME }}
+      NPM_BUILD: ${{ matrix.NPM_BUILD }}
+      BUILD_ARTIFACT: ${{ matrix.BUILD_ARTIFACT }}
+    secrets:
+      DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }}
+      ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }}
+      RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}
+
+  build-dockers-github-activity-tracker-service:
+    needs: build-github-activity-tracker-service
+    strategy:
+      matrix:
+        include:
+          - SERVICE_LOCATION: './github-activity-tracker/backend'
+            SERVICE_NAME: 'github-activity-tracker-service'
+            NPM_BUILD: true
+            BUILD_ARTIFACT: 'github-activity-tracker-service'
+      fail-fast: false
+    name: ${{ matrix.SERVICE_NAME }}
+    uses: mosip/kattu/.github/workflows/docker-build.yml@master-java21
+    with:
+      SERVICE_LOCATION: ${{ matrix.SERVICE_LOCATION }}
+      SERVICE_NAME: ${{ matrix.SERVICE_NAME }}
+      NPM_BUILD: ${{ matrix.NPM_BUILD }}
+      BUILD_ARTIFACT: ${{ matrix.BUILD_ARTIFACT }}
+    secrets:
+      DEV_NAMESPACE_DOCKER_HUB: ${{ secrets.DEV_NAMESPACE_DOCKER_HUB }}
+      ACTOR_DOCKER_HUB: ${{ secrets.ACTOR_DOCKER_HUB }}
+      RELEASE_DOCKER_HUB: ${{ secrets.RELEASE_DOCKER_HUB }}
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK }}

MosipNexus/.dockerignore

@@ -0,0 +1,40 @@
+# Secrets — never bake into image
+.env
+.env.*
+
+# Crawled data files — large (100–400 MB) and not needed by API/UI pods.
+# The updater CronJob writes fresh data to a mounted PVC at runtime.
+data/
+
+# Confluence data — contains internal credentials, must never be in image
+data/mosip_confluence.json
+
+# Vector store — pgvector runs externally; no local store needed
+chroma_db/
+
+# Python artifacts
+.venv/
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# VCS
+.git/
+.gitignore
+
+# Dev/test
+tests/
+*.test.py
+
+# IDE
+.vscode/
+.idea/
+
+# Docs (not needed at runtime)
+docs/*.pptx
+docs/*.docx
+
+# OS
+.DS_Store
+Thumbs.db

MosipNexus/.env.example

@@ -0,0 +1,41 @@
+# ── Required ───────────────────────────────────────────────────────────────────
+GROQ_API_KEY=YOUR_GROQ_API_KEY         # https://console.groq.com (free tier)
+
+# ── Optional: HuggingFace token (avoids rate-limiting on model downloads) ──────
+HF_TOKEN=hf_your_token_here
+
+# ── Optional: LangSmith observability (free tier — 5k traces/month) ────────────
+# Sign up at https://smith.langchain.com, create a project named "mosip-nexus"
+LANGCHAIN_TRACING_V2=true
+LANGCHAIN_API_KEY=lsv2_pt_your_key_here
+LANGCHAIN_PROJECT=mosip-nexus
+
+# ── Optional: GitHub crawler (strongly recommended — avoids 60 req/hr limit) ───
+# Create at: github.com → Settings → Developer Settings → Personal Access Tokens → Classic
+# No special scopes needed (MOSIP repos are public).
+GITHUB_TOKEN=ghp_your_token_here
+
+# ── Optional: Confluence crawler ────────────────────────────────────────────────
+# Generate Atlassian API token at: id.atlassian.com/manage-profile/security/api-tokens
+CONFLUENCE_URL=https://yourorg.atlassian.net/wiki
+CONFLUENCE_USER=your_email@example.com
+CONFLUENCE_TOKEN=your_atlassian_api_token
+CONFLUENCE_SPACE_KEYS=MOSIP,MOSIPDEV    # comma-separated space keys
+
+# ── Optional: Jira crawler ──────────────────────────────────────────────────────
+JIRA_URL=https://yourorg.atlassian.net
+JIRA_USER=your_email@example.com
+JIRA_TOKEN=your_atlassian_api_token
+JIRA_PROJECT_KEYS=MOSIP,MISP            # comma-separated project keys
+
+# ── Optional: Email notifications ───────────────────────────────────────────────
+# Gmail: generate an App Password at Google Account → Security → App passwords
+# Port 465 = SSL  |  Port 587 = STARTTLS (default for most providers)
+SMTP_HOST=smtp.gmail.com
+SMTP_PORT=587
+SMTP_USER=your_gmail@gmail.com
+SMTP_PASSWORD=your_app_password_here
+NOTIFY_EMAIL=mosip-team@example.com     # recipient address
+
+YOUTUBE_API_KEY=AIzaSyDrb2q9RxYKHH3nzG9j-X_Ot5-VAYG_5n4
+YOUTUBE_CHANNEL_HANDLE=mosip16

MosipNexus/.gitignore

@@ -0,0 +1,43 @@
+# ── Secrets ───────────────────────────────────────────────────────────────────
+.env
+.env.*
+!.env.example
+
+# ── Python ────────────────────────────────────────────────────────────────────
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+
+# ── Vector database (generated — rebuild with ingestion/store.py) ─────────────
+chroma_db/
+
+# ── Crawled data — JSON files ARE committed so cloners skip the 10-hour crawl
+# Exceptions: crawl_state (runtime), confluence and jira data (may contain internal secrets)
+data/crawl_state.json
+data/mosip_confluence.json
+data/mosip_jira.json
+
+# ── pg_dump / zip exports — large binaries, not for git ──────────────────────
+nexus_vectors.dump
+nexus_vectors.zip
+
+# ── Virtual environment (managed at monorepo root) ────────────────────────────
+.venv/
+
+# ── IDE ───────────────────────────────────────────────────────────────────────
+.vscode/
+.idea/
+
+# ── Logs ──────────────────────────────────────────────────────────────────────
+*.log
+logs/
+
+# ── Test cache ────────────────────────────────────────────────────────────────
+.pytest_cache/
+.coverage
+htmlcov/
+
+# ── OS ────────────────────────────────────────────────────────────────────────
+.DS_Store
+Thumbs.db

MosipNexus/.python-version

@@ -0,0 +1 @@
+3.13

MosipNexus/.streamlit/config.toml

@@ -0,0 +1,8 @@
+[server]
+headless = true
+port = 8501
+enableCORS = false
+enableXsrfProtection = false
+
+[browser]
+gatherUsageStats = false

MosipNexus/Dockerfile

@@ -0,0 +1,38 @@
+FROM python:3.13-slim
+
+# libgomp1 — required by PyTorch (sentence-transformers)
+# curl     — used by K8s liveness probes
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libgomp1 \
+        curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv (Python package manager)
+RUN pip install --no-cache-dir uv
+
+WORKDIR /app/MosipNexus
+
+# --- Dependency layer (cached unless pyproject.toml changes) ---
+COPY pyproject.toml ./
+# Generate uv.lock if not present; use it if it is (reproducible builds).
+# Run `uv lock && git add uv.lock` before building to pin exact versions.
+RUN uv sync
+
+# --- Application source (data/ and secrets excluded by .dockerignore) ---
+COPY . .
+
+# --- Pre-cache the HuggingFace embedding model (~1.1 GB) ---
+# This bakes the model into the image so pods start with zero download delay.
+# The model is stored at /root/.cache/huggingface inside the image.
+RUN uv run python -c \
+    "from langchain_huggingface import HuggingFaceEmbeddings; \
+     HuggingFaceEmbeddings(model_name='intfloat/multilingual-e5-base'); \
+     print('HF model cached.')"
+
+# PYTHONPATH ensures imports resolve from within /app/MosipNexus
+ENV PYTHONPATH=/app/MosipNexus
+
+EXPOSE 8000 8501
+
+# Default: API server. K8s deployment-ui.yaml overrides this with streamlit.
+CMD ["uv", "run", "uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]