DEV-1185: fix AWG download of related files

.github/workflows/ci.yaml

@@ -5,9 +5,9 @@ on:
       - develop
       - release/**
       - hotfix/**
-      - feat/dev-583-remove-set-env-in-github-actions
+      - feat/DEV-1185-fix-awg-download-related-files
     tags:
-      - "*"
+      - '*'
 
 jobs:
   build:
@@ -16,34 +16,34 @@ jobs:
       matrix:
         os:
           - macos-10.15
-          - ubuntu-20.04
-          - ubuntu-16.04
-          - ubuntu-18.04
-          - windows-latest
+          #- ubuntu-20.04
+          #- ubuntu-16.04
+          #- ubuntu-18.04
+          #- windows-latest
         python:
           - 3.7
     env:
-      GDC_CLIENT_ZIP: "gdc-client.zip"
+      GDC_CLIENT_ZIP: 'gdc-client.zip'
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python }}
-    - name: Run Tests
-      if: matrix.os != 'windows-latest'
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox
-        tox -e py
-    - name: Package
-      run: |
-        cd bin
-        pip install virtualenv
-        . ./package
-        echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV
-      shell: bash
-    - uses: actions/upload-artifact@v2
-      with:
-        name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }}
-        path: bin/gdc-client_*.zip
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Run Tests
+        if: matrix.os != 'windows-latest'
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+          tox -e py
+      - name: Package
+        run: |
+          cd bin
+          pip install virtualenv
+          . ./package
+          echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV
+        shell: bash
+      - uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }}
+          path: bin/gdc-client_*.zip

dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
 #    pip-compile dev-requirements.in
@@ -14,12 +14,12 @@ aws-sam-translator==1.11.0
     #   cfn-lint
 aws-xray-sdk==2.11.0
     # via moto
+boto==2.49.0
+    # via moto
 boto3==1.16.63
     # via
     #   aws-sam-translator
     #   moto
-boto==2.49.0
-    # via moto
 botocore==1.19.63
     # via
     #   aws-xray-sdk
@@ -68,8 +68,9 @@ idna==2.8
     #   -c requirements.txt
     #   moto
     #   requests
-importlib-metadata==2.1.3
+importlib-metadata==6.7.0 ; python_version < "3.8"
     # via
+    #   -c requirements.txt
     #   pluggy
     #   pytest
 itsdangerous==1.1.0
@@ -107,8 +108,6 @@ moto==1.3.16
     # via -r dev-requirements.in
 packaging==20.9
     # via pytest
-pathlib2==2.3.7.post1
-    # via pytest
 pluggy==0.13.1
     # via pytest
 py==1.11.0
@@ -124,12 +123,12 @@ pycparser==2.21
     #   cffi
 pyparsing==2.4.7
     # via packaging
-pytest-cov==2.7.1
-    # via -r dev-requirements.in
 pytest==4.6.2
     # via
     #   -r dev-requirements.in
     #   pytest-cov
+pytest-cov==2.7.1
+    # via -r dev-requirements.in
 python-dateutil==2.8.2
     # via
     #   botocore
@@ -145,8 +144,6 @@ pyyaml==5.3.1
     #   -c requirements.txt
     #   cfn-lint
     #   moto
-requests-mock==1.5.2
-    # via -r dev-requirements.in
 requests==2.22.0
     # via
     #   -c requirements.txt
@@ -156,6 +153,8 @@ requests==2.22.0
     #   moto
     #   requests-mock
     #   responses
+requests-mock==1.5.2
+    # via -r dev-requirements.in
 responses==0.17.0
     # via moto
 rsa==4.7.2
@@ -172,7 +171,6 @@ six==1.16.0
     #   ecdsa
     #   mock
     #   moto
-    #   pathlib2
     #   pytest
     #   python-dateutil
     #   python-jose
@@ -181,6 +179,10 @@ six==1.16.0
     #   websocket-client
 sshpubkeys==3.3.1
     # via moto
+typing-extensions==4.7.1
+    # via
+    #   -c requirements.txt
+    #   importlib-metadata
 urllib3==1.25.11
     # via
     #   -c requirements.txt
@@ -199,8 +201,9 @@ wrapt==1.14.1
     # via aws-xray-sdk
 xmltodict==0.13.0
     # via moto
-zipp==1.2.0
+zipp==3.15.0
     # via
+    #   -c requirements.txt
     #   importlib-metadata
     #   moto
 

gdc_client/download/parser.py

@@ -80,7 +80,9 @@ def download(parser, args):
 
     ids = ids_map.values() if args.latest else ids_map.keys()
 
-    index_client = GDCIndexClient(args.server, not args.no_verify)
+    index_client = GDCIndexClient(
+        uri=args.server, token=args.token_file, verify=not args.no_verify
+    )
     client = get_client(args, index_client)
 
     # separate the smaller files from the larger files
@@ -163,7 +165,6 @@ def download(parser, args):
 
 
 def retry_download(client, url, retry_amount, no_auto_retry, wait_time):
-
     log.debug("Retrying download {0}".format(url))
 
     error = True

gdc_client/query/index.py

@@ -8,12 +8,13 @@
 
 
 class GDCIndexClient(object):
-    def __init__(self, uri, verify=True):
+    def __init__(self, uri, token=None, verify=True):
         self.uri = uri
         self.active_meta_endpoint = "/v0/files"
         self.legacy_meta_endpoint = "/v0/legacy/files"
         self.metadata = dict()
         self.verify = verify
+        self.token = token
 
     def get_related_files(self, uuid):
         # type: (str) -> list[str]
@@ -54,7 +55,8 @@ def _get_hits(self, url, metadata_query):
         """
         json_response = {}
         # using a POST request lets us avoid the MAX URL character length limit
-        r = requests.post(url, json=metadata_query, verify=self.verify)
+        headers = {"X-Auth-Token": self.token}
+        r = requests.post(url, json=metadata_query, headers=headers, verify=self.verify)
 
         if r is None:
             return []

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile
+# This file is autogenerated by pip-compile with python 3.7
 # To update, run:
 #
 #    pip-compile
@@ -12,36 +12,38 @@ chardet==3.0.4
     # via requests
 cryptography==2.8
     # via
-    #   gdc_client (setup.py)
+    #   gdc-client (setup.py)
     #   pyopenssl
 idna==2.8
     # via requests
+importlib-metadata==6.7.0 ; python_version < "3.8"
+    # via gdc-client (setup.py)
 intervaltree==3.0.2
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 jsonschema==2.6.0
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 lxml==4.4.2
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 ndg-httpsclient==0.5.0
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 progressbar2==3.43.1
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 pyasn1==0.4.3
     # via
-    #   gdc_client (setup.py)
+    #   gdc-client (setup.py)
     #   ndg-httpsclient
 pycparser==2.21
     # via cffi
 pyopenssl==18.0.0
     # via
-    #   gdc_client (setup.py)
+    #   gdc-client (setup.py)
     #   ndg-httpsclient
 python-utils==2.7.1
     # via progressbar2
 pyyaml==5.3.1
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 requests==2.22.0
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
 six==1.16.0
     # via
     #   cryptography
@@ -51,6 +53,10 @@ six==1.16.0
 sortedcontainers==2.4.0
     # via intervaltree
 termcolor==1.1.0
-    # via gdc_client (setup.py)
+    # via gdc-client (setup.py)
+typing-extensions==4.7.1
+    # via importlib-metadata
 urllib3==1.25.11
     # via requests
+zipp==3.15.0
+    # via importlib-metadata