Skip to content

Commit

Permalink
Merge pull request #1315 from manics/zenodo
Browse files Browse the repository at this point in the history 
Get Zenodo working again
  • Loading branch information
@yuvipanda
yuvipanda committed Oct 19, 2023
2 parents de496f8 + fc7de27 commit aa57e01
Show file tree
Hide file tree
Showing 4 changed files with 101 additions and 25 deletions.
12 changes: 9 additions & 3 deletions repo2docker/contentproviders/doi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,15 @@ def doi2url(self, doi):
try:
resp = self._request(f"https://doi.org/{doi}")
resp.raise_for_status()
# If the DOI doesn't resolve, just return URL
except HTTPError:
return doi
except HTTPError as e:
# If the DOI doesn't exist, just return URL
if e.response.status_code == 404:
return doi
# Reraise any other errors because if the DOI service is down (or
# we hit a rate limit) we don't want to silently continue to the
# default Git provider as this leads to a misleading error.
logging.error(f"DOI {doi} does not resolve: {e}")
raise
return resp.url
else:
# Just return what is actulally just a URL
Expand Down
32 changes: 24 additions & 8 deletions repo2docker/contentproviders/zenodo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,26 @@ def __init__(self):
"hostname": [
"https://sandbox.zenodo.org/record/",
"http://sandbox.zenodo.org/record/",
"http://sandbox.zenodo.org/records/",
],
"api": "https://sandbox.zenodo.org/api/records/",
"filepath": "files",
"filename": "filename",
"download": "links.download",
"files": "links.files",
"filepath": "entries",
"filename": "key",
"download": "links.content",
"type": "metadata.upload_type",
},
{
"hostname": ["https://zenodo.org/record/", "http://zenodo.org/record/"],
"hostname": [
"https://zenodo.org/record/",
"http://zenodo.org/record/",
"https://zenodo.org/records/",
],
"api": "https://zenodo.org/api/records/",
"filepath": "files",
"filename": "filename",
"download": "links.download",
"files": "links.files",
"filepath": "entries",
"filename": "key",
"download": "links.content",
"type": "metadata.upload_type",
},
{
Expand All @@ -43,6 +50,7 @@ def __init__(self):
"http://data.caltech.edu/records/",
],
"api": "https://data.caltech.edu/api/record/",
"files": "",
"filepath": "metadata.electronic_location_and_access",
"filename": "electronic_name.0",
"download": "uniform_resource_identifier",
Expand All @@ -69,9 +77,17 @@ def fetch(self, spec, output_dir, yield_output=False):
f'{host["api"]}{record_id}',
headers={"accept": "application/json"},
)

record = resp.json()

if host["files"]:
yield f"Fetching Zenodo record {record_id} files.\n"
files_url = deep_get(record, host["files"])
resp = self.urlopen(
files_url,
headers={"accept": "application/json"},
)
record = resp.json()

files = deep_get(record, host["filepath"])
only_one_file = len(files) == 1
for file_ref in files:
Expand Down
14 changes: 10 additions & 4 deletions tests/unit/contentproviders/test_doi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,14 @@ def test_url_headers(requests_mock):
assert result.request.headers["User-Agent"] == f"repo2docker {__version__}"


def test_unresolving_doi():
@pytest.mark.parametrize(
"requested_doi, expected",
[
("10.5281/zenodo.3242074", "https://zenodo.org/records/3242074"),
# Unresolving DOI:
("10.1/1234", "10.1/1234"),
],
)
def test_doi2url(requested_doi, expected):
doi = DoiProvider()

fakedoi = "10.1/1234"
assert doi.doi2url(fakedoi) is fakedoi
assert doi.doi2url(requested_doi) == expected
68 changes: 58 additions & 10 deletions tests/unit/contentproviders/test_zenodo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,31 @@ def test_fetch_software_from_github_archive(requests_mock):
# we "fetch" a local ZIP file to simulate a Zenodo record created from a
# GitHub repository via the Zenodo-GitHub integration
with zenodo_archive() as zen_path:
mock_response = {
mock_record = {
"files": [
{
"filename": "some_dir/afake.zip",
"links": {"download": f"file://{zen_path}"},
}
],
"links": {
"files": "https://zenodo.org/api/records/1234/files",
},
"metadata": {"upload_type": "other"},
}
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)

mock_record_files = {
"entries": [
{
"key": "some_dir/afake.zip",
"links": {"content": f"file://{zen_path}"},
}
],
}
requests_mock.get(
"https://zenodo.org/api/records/1234/files", json=mock_record_files
)

requests_mock.get(f"file://{zen_path}", content=open(zen_path, "rb").read())

zen = Zenodo()
Expand All @@ -121,18 +136,33 @@ def test_fetch_software(requests_mock):
# we "fetch" a local ZIP file to simulate a Zenodo software record with a
# ZIP file in it
with zenodo_archive() as zen_path:
mock_response = {
mock_record = {
"files": [
{
# this is the difference to the GitHub generated one,
# the ZIP file isn't in a directory
"filename": "afake.zip",
"links": {"download": f"file://{zen_path}"},
}
],
"links": {
"files": "https://zenodo.org/api/records/1234/files",
},
"metadata": {"upload_type": "software"},
}
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)

mock_record_files = {
"entries": [
{
"key": "afake.zip",
"links": {"content": f"file://{zen_path}"},
}
],
}
requests_mock.get(
"https://zenodo.org/api/records/1234/files", json=mock_record_files
)

requests_mock.get(f"file://{zen_path}", content=open(zen_path, "rb").read())

with TemporaryDirectory() as d:
Expand All @@ -151,20 +181,38 @@ def test_fetch_data(requests_mock):
# we "fetch" a local ZIP file to simulate a Zenodo data record
with zenodo_archive() as a_zen_path:
with zenodo_archive() as b_zen_path:
mock_response = {
mock_record = {
"files": [
{
"filename": "afake.zip",
"links": {"download": f"file://{a_zen_path}"},
},
{
"filename": "bfake.zip",
"links": {"download": f"file://{b_zen_path}"},
},
],
"links": {
"files": "https://zenodo.org/api/records/1234/files",
},
"metadata": {"upload_type": "data"},
}
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_response)
requests_mock.get("https://zenodo.org/api/records/1234", json=mock_record)

mock_record_files = {
"entries": [
{
"key": "afake.zip",
"links": {"content": f"file://{a_zen_path}"},
},
{
"key": "bfake.zip",
"links": {"content": f"file://{b_zen_path}"},
},
],
}
requests_mock.get(
"https://zenodo.org/api/records/1234/files", json=mock_record_files
)

requests_mock.get(
f"file://{a_zen_path}", content=open(a_zen_path, "rb").read()
)
Expand Down

0 comments on commit aa57e01

Please sign in to comment.