diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 00d6e5ab..787b4103 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,9 +5,9 @@ on: - develop - release/** - hotfix/** - - feat/dev-583-remove-set-env-in-github-actions + - feat/DEV-1185-fix-awg-download-related-files tags: - - "*" + - '*' jobs: build: @@ -16,34 +16,34 @@ jobs: matrix: os: - macos-10.15 - - ubuntu-20.04 - - ubuntu-16.04 - - ubuntu-18.04 - - windows-latest + #- ubuntu-20.04 + #- ubuntu-16.04 + #- ubuntu-18.04 + #- windows-latest python: - 3.7 env: - GDC_CLIENT_ZIP: "gdc-client.zip" + GDC_CLIENT_ZIP: 'gdc-client.zip' steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - name: Run Tests - if: matrix.os != 'windows-latest' - run: | - python -m pip install --upgrade pip - pip install tox - tox -e py - - name: Package - run: | - cd bin - pip install virtualenv - . ./package - echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV - shell: bash - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }} - path: bin/gdc-client_*.zip + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Run Tests + if: matrix.os != 'windows-latest' + run: | + python -m pip install --upgrade pip + pip install tox + tox -e py + - name: Package + run: | + cd bin + pip install virtualenv + . ./package + echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV + shell: bash + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }} + path: bin/gdc-client_*.zip diff --git a/dev-requirements.txt b/dev-requirements.txt index 293e51ee..e084704c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile dev-requirements.in @@ -14,12 +14,12 @@ aws-sam-translator==1.11.0 # cfn-lint aws-xray-sdk==2.11.0 # via moto +boto==2.49.0 + # via moto boto3==1.16.63 # via # aws-sam-translator # moto -boto==2.49.0 - # via moto botocore==1.19.63 # via # aws-xray-sdk @@ -68,8 +68,9 @@ idna==2.8 # -c requirements.txt # moto # requests -importlib-metadata==2.1.3 +importlib-metadata==6.7.0 ; python_version < "3.8" # via + # -c requirements.txt # pluggy # pytest itsdangerous==1.1.0 @@ -107,8 +108,6 @@ moto==1.3.16 # via -r dev-requirements.in packaging==20.9 # via pytest -pathlib2==2.3.7.post1 - # via pytest pluggy==0.13.1 # via pytest py==1.11.0 @@ -124,12 +123,12 @@ pycparser==2.21 # cffi pyparsing==2.4.7 # via packaging -pytest-cov==2.7.1 - # via -r dev-requirements.in pytest==4.6.2 # via # -r dev-requirements.in # pytest-cov +pytest-cov==2.7.1 + # via -r dev-requirements.in python-dateutil==2.8.2 # via # botocore @@ -145,8 +144,6 @@ pyyaml==5.3.1 # -c requirements.txt # cfn-lint # moto -requests-mock==1.5.2 - # via -r dev-requirements.in requests==2.22.0 # via # -c requirements.txt @@ -156,6 +153,8 @@ requests==2.22.0 # moto # requests-mock # responses +requests-mock==1.5.2 + # via -r dev-requirements.in responses==0.17.0 # via moto rsa==4.7.2 @@ -172,7 +171,6 @@ six==1.16.0 # ecdsa # mock # moto - # pathlib2 # pytest # python-dateutil # python-jose @@ -181,6 +179,10 @@ six==1.16.0 # websocket-client sshpubkeys==3.3.1 # via moto +typing-extensions==4.7.1 + # via + # -c requirements.txt + # importlib-metadata urllib3==1.25.11 # via # -c requirements.txt @@ -199,8 +201,9 @@ wrapt==1.14.1 # via aws-xray-sdk xmltodict==0.13.0 # via moto -zipp==1.2.0 +zipp==3.15.0 # via + # -c requirements.txt # importlib-metadata # moto diff --git a/gdc_client/download/parser.py b/gdc_client/download/parser.py index e66ac156..15987b61 100644 --- a/gdc_client/download/parser.py +++ b/gdc_client/download/parser.py @@ -80,7 +80,9 @@ def download(parser, args): ids = ids_map.values() if args.latest else ids_map.keys() - index_client = GDCIndexClient(args.server, not args.no_verify) + index_client = GDCIndexClient( + uri=args.server, token=args.token_file, verify=not args.no_verify + ) client = get_client(args, index_client) # separate the smaller files from the larger files @@ -163,7 +165,6 @@ def download(parser, args): def retry_download(client, url, retry_amount, no_auto_retry, wait_time): - log.debug("Retrying download {0}".format(url)) error = True diff --git a/gdc_client/query/index.py b/gdc_client/query/index.py index 2045e3e3..dc6983a0 100644 --- a/gdc_client/query/index.py +++ b/gdc_client/query/index.py @@ -8,12 +8,13 @@ class GDCIndexClient(object): - def __init__(self, uri, verify=True): + def __init__(self, uri, token=None, verify=True): self.uri = uri self.active_meta_endpoint = "/v0/files" self.legacy_meta_endpoint = "/v0/legacy/files" self.metadata = dict() self.verify = verify + self.token = token def get_related_files(self, uuid): # type: (str) -> list[str] @@ -54,7 +55,8 @@ def _get_hits(self, url, metadata_query): """ json_response = {} # using a POST request lets us avoid the MAX URL character length limit - r = requests.post(url, json=metadata_query, verify=self.verify) + headers = {"X-Auth-Token": self.token} + r = requests.post(url, json=metadata_query, headers=headers, verify=self.verify) if r is None: return [] diff --git a/requirements.txt b/requirements.txt index 92e0db53..ccffc7c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile @@ -12,36 +12,38 @@ chardet==3.0.4 # via requests cryptography==2.8 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # pyopenssl idna==2.8 # via requests +importlib-metadata==6.7.0 ; python_version < "3.8" + # via gdc-client (setup.py) intervaltree==3.0.2 - # via gdc_client (setup.py) + # via gdc-client (setup.py) jsonschema==2.6.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) lxml==4.4.2 - # via gdc_client (setup.py) + # via gdc-client (setup.py) ndg-httpsclient==0.5.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) progressbar2==3.43.1 - # via gdc_client (setup.py) + # via gdc-client (setup.py) pyasn1==0.4.3 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # ndg-httpsclient pycparser==2.21 # via cffi pyopenssl==18.0.0 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # ndg-httpsclient python-utils==2.7.1 # via progressbar2 pyyaml==5.3.1 - # via gdc_client (setup.py) + # via gdc-client (setup.py) requests==2.22.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) six==1.16.0 # via # cryptography @@ -51,6 +53,10 @@ six==1.16.0 sortedcontainers==2.4.0 # via intervaltree termcolor==1.1.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) +typing-extensions==4.7.1 + # via importlib-metadata urllib3==1.25.11 # via requests +zipp==3.15.0 + # via importlib-metadata