From 77202828d4f5eb87c1df6594628bb1a8c4c4fbde Mon Sep 17 00:00:00 2001 From: jiakf Date: Wed, 24 Jan 2024 12:49:23 -0600 Subject: [PATCH 1/3] DEV-1185: fix AWG download of related files Previously, requests for file metadata in the AWG environment were unauthenticated. However, the metadata endpoint in AWG requires proper authenticated requests. This change allows for requests for file metadata in the AWG environment to be properly authenticated. --- gdc_client/download/parser.py | 5 +++-- gdc_client/query/index.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/gdc_client/download/parser.py b/gdc_client/download/parser.py index e66ac156..15987b61 100644 --- a/gdc_client/download/parser.py +++ b/gdc_client/download/parser.py @@ -80,7 +80,9 @@ def download(parser, args): ids = ids_map.values() if args.latest else ids_map.keys() - index_client = GDCIndexClient(args.server, not args.no_verify) + index_client = GDCIndexClient( + uri=args.server, token=args.token_file, verify=not args.no_verify + ) client = get_client(args, index_client) # separate the smaller files from the larger files @@ -163,7 +165,6 @@ def download(parser, args): def retry_download(client, url, retry_amount, no_auto_retry, wait_time): - log.debug("Retrying download {0}".format(url)) error = True diff --git a/gdc_client/query/index.py b/gdc_client/query/index.py index 2045e3e3..dc6983a0 100644 --- a/gdc_client/query/index.py +++ b/gdc_client/query/index.py @@ -8,12 +8,13 @@ class GDCIndexClient(object): - def __init__(self, uri, verify=True): + def __init__(self, uri, token=None, verify=True): self.uri = uri self.active_meta_endpoint = "/v0/files" self.legacy_meta_endpoint = "/v0/legacy/files" self.metadata = dict() self.verify = verify + self.token = token def get_related_files(self, uuid): # type: (str) -> list[str] @@ -54,7 +55,8 @@ def _get_hits(self, url, metadata_query): """ json_response = {} # using a POST request lets us avoid the MAX URL character length limit - r = requests.post(url, json=metadata_query, verify=self.verify) + headers = {"X-Auth-Token": self.token} + r = requests.post(url, json=metadata_query, headers=headers, verify=self.verify) if r is None: return [] From 86c5d566274f4775713ebd870afe01dacb57b4d3 Mon Sep 17 00:00:00 2001 From: jiakf Date: Tue, 6 Feb 2024 10:09:34 -0600 Subject: [PATCH 2/3] update temporariliy github actions --- .github/workflows/ci.yaml | 58 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 00d6e5ab..787b4103 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,9 +5,9 @@ on: - develop - release/** - hotfix/** - - feat/dev-583-remove-set-env-in-github-actions + - feat/DEV-1185-fix-awg-download-related-files tags: - - "*" + - '*' jobs: build: @@ -16,34 +16,34 @@ jobs: matrix: os: - macos-10.15 - - ubuntu-20.04 - - ubuntu-16.04 - - ubuntu-18.04 - - windows-latest + #- ubuntu-20.04 + #- ubuntu-16.04 + #- ubuntu-18.04 + #- windows-latest python: - 3.7 env: - GDC_CLIENT_ZIP: "gdc-client.zip" + GDC_CLIENT_ZIP: 'gdc-client.zip' steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python }} - - name: Run Tests - if: matrix.os != 'windows-latest' - run: | - python -m pip install --upgrade pip - pip install tox - tox -e py - - name: Package - run: | - cd bin - pip install virtualenv - . ./package - echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV - shell: bash - - uses: actions/upload-artifact@v2 - with: - name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }} - path: bin/gdc-client_*.zip + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python }} + - name: Run Tests + if: matrix.os != 'windows-latest' + run: | + python -m pip install --upgrade pip + pip install tox + tox -e py + - name: Package + run: | + cd bin + pip install virtualenv + . ./package + echo "GDC_CLIENT_ZIP=$GDC_CLIENT_ZIP" >> $GITHUB_ENV + shell: bash + - uses: actions/upload-artifact@v2 + with: + name: ${{ env.GDC_CLIENT_ZIP }}-py${{ matrix.python }}-${{ matrix.os }} + path: bin/gdc-client_*.zip From 3bf21fbd84fa4d03c778e55b860e212297fc8338 Mon Sep 17 00:00:00 2001 From: jiakf Date: Tue, 6 Feb 2024 11:10:51 -0600 Subject: [PATCH 3/3] update requirements.txt files --- dev-requirements.txt | 27 +++++++++++++++------------ requirements.txt | 30 ++++++++++++++++++------------ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 293e51ee..e084704c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile dev-requirements.in @@ -14,12 +14,12 @@ aws-sam-translator==1.11.0 # cfn-lint aws-xray-sdk==2.11.0 # via moto +boto==2.49.0 + # via moto boto3==1.16.63 # via # aws-sam-translator # moto -boto==2.49.0 - # via moto botocore==1.19.63 # via # aws-xray-sdk @@ -68,8 +68,9 @@ idna==2.8 # -c requirements.txt # moto # requests -importlib-metadata==2.1.3 +importlib-metadata==6.7.0 ; python_version < "3.8" # via + # -c requirements.txt # pluggy # pytest itsdangerous==1.1.0 @@ -107,8 +108,6 @@ moto==1.3.16 # via -r dev-requirements.in packaging==20.9 # via pytest -pathlib2==2.3.7.post1 - # via pytest pluggy==0.13.1 # via pytest py==1.11.0 @@ -124,12 +123,12 @@ pycparser==2.21 # cffi pyparsing==2.4.7 # via packaging -pytest-cov==2.7.1 - # via -r dev-requirements.in pytest==4.6.2 # via # -r dev-requirements.in # pytest-cov +pytest-cov==2.7.1 + # via -r dev-requirements.in python-dateutil==2.8.2 # via # botocore @@ -145,8 +144,6 @@ pyyaml==5.3.1 # -c requirements.txt # cfn-lint # moto -requests-mock==1.5.2 - # via -r dev-requirements.in requests==2.22.0 # via # -c requirements.txt @@ -156,6 +153,8 @@ requests==2.22.0 # moto # requests-mock # responses +requests-mock==1.5.2 + # via -r dev-requirements.in responses==0.17.0 # via moto rsa==4.7.2 @@ -172,7 +171,6 @@ six==1.16.0 # ecdsa # mock # moto - # pathlib2 # pytest # python-dateutil # python-jose @@ -181,6 +179,10 @@ six==1.16.0 # websocket-client sshpubkeys==3.3.1 # via moto +typing-extensions==4.7.1 + # via + # -c requirements.txt + # importlib-metadata urllib3==1.25.11 # via # -c requirements.txt @@ -199,8 +201,9 @@ wrapt==1.14.1 # via aws-xray-sdk xmltodict==0.13.0 # via moto -zipp==1.2.0 +zipp==3.15.0 # via + # -c requirements.txt # importlib-metadata # moto diff --git a/requirements.txt b/requirements.txt index 92e0db53..ccffc7c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile +# This file is autogenerated by pip-compile with python 3.7 # To update, run: # # pip-compile @@ -12,36 +12,38 @@ chardet==3.0.4 # via requests cryptography==2.8 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # pyopenssl idna==2.8 # via requests +importlib-metadata==6.7.0 ; python_version < "3.8" + # via gdc-client (setup.py) intervaltree==3.0.2 - # via gdc_client (setup.py) + # via gdc-client (setup.py) jsonschema==2.6.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) lxml==4.4.2 - # via gdc_client (setup.py) + # via gdc-client (setup.py) ndg-httpsclient==0.5.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) progressbar2==3.43.1 - # via gdc_client (setup.py) + # via gdc-client (setup.py) pyasn1==0.4.3 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # ndg-httpsclient pycparser==2.21 # via cffi pyopenssl==18.0.0 # via - # gdc_client (setup.py) + # gdc-client (setup.py) # ndg-httpsclient python-utils==2.7.1 # via progressbar2 pyyaml==5.3.1 - # via gdc_client (setup.py) + # via gdc-client (setup.py) requests==2.22.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) six==1.16.0 # via # cryptography @@ -51,6 +53,10 @@ six==1.16.0 sortedcontainers==2.4.0 # via intervaltree termcolor==1.1.0 - # via gdc_client (setup.py) + # via gdc-client (setup.py) +typing-extensions==4.7.1 + # via importlib-metadata urllib3==1.25.11 # via requests +zipp==3.15.0 + # via importlib-metadata