Skip to content

Commit

Permalink
Merge pull request #1258 from firedrakeproject/doi-install
Browse files Browse the repository at this point in the history 
* doi-install:
  install: Add DOI resolution test mode
  install: Implement firedrake-install --doi XXX
  • Loading branch information
@wence-
wence- committed Aug 2, 2018
2 parents d3f2346 + 35de95b commit 1951865
Show file tree
Hide file tree
Showing 3 changed files with 117 additions and 20 deletions.
7 changes: 7 additions & 0 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,13 @@ cd firedrake/src/pyadjoint; python -m pytest -v tests/firedrake_adjoint
}
}
}
stage('Zenodo API canary') {
steps {
timestamps {
sh 'scripts/firedrake-install --test-doi-resolution || (cat firedrake-install.log && /bin/false)'
}
}
}
}
}

16 changes: 16 additions & 0 deletions docs/source/zenodo.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@ with::

firedrake-zenodo -h

Installing an archived release
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:doc:`firedrake-install <download>` has support for installing a
Zenodo-archived release. If you have a DOI for a particular Zenodo
release, you can install the matching set of components with::

firedrake-install --doi MY_ZENODO_DOI

.. note::

``firedrake-update`` will not work out of the box in this scenario,
because the components are checked out in a `detached head
<https://www.git-tower.com/learn/git/faq/detached-head-when-checkout-commit>`_
state.

What else do you need to do?
----------------------------

Expand Down
114 changes: 94 additions & 20 deletions scripts/firedrake-install
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import os
import shutil
from glob import glob
from argparse import ArgumentParser, RawDescriptionHelpFormatter
import argparse
from collections import OrderedDict
import atexit

Expand Down Expand Up @@ -76,6 +77,24 @@ else:
sys.exit("Script must be invoked either as firedrake-install or firedrake-update")


# Set up logging
# Log to file at DEBUG level
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)-6s %(message)s',
filename='firedrake-%s.log' % mode,
filemode='w')
# Log to console at INFO level
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)

log = logging.getLogger()

log.info("Running %s" % " ".join(sys.argv))


if sys.version_info < (3, 5):
if mode == "install":
print("""\nInstalling Firedrake requires Python 3, at least version 3.5.
Expand All @@ -100,6 +119,53 @@ jenkins = "JENKINS_URL" in os.environ
ci_testing_firedrake = "FIREDRAKE_CI_TESTS" in os.environ


def resolve_doi_branches(doi):
import requests
import hashlib
log.info("Installing Firedrake components specified by {}".format(doi))
response = requests.get("https://zenodo.org/api/records",
params={"q": "doi:{}".format(doi.replace("/", r"\/"))})
if response.status_code >= 400:
log.error("Unable to obtain Zenodo record for doi {}".format(doi))
log.error("Response was {}".format(response.json()))
sys.exit(1)
response = response.json()
try:
record, = response["hits"]["hits"]
except ValueError:
log.error("Was expecting one record for doi '{doi}', found {num}".format(
doi=doi, num=response["hits"]["total"]))
log.error("Response was {}".format(response))
sys.exit(1)
files = record["files"]
try:
componentjson, = (f for f in files if f["key"] == "components.json")
except ValueError:
log.error("Expecting to find exactly one 'components.json' in record")
sys.exit(1)
download = requests.get(componentjson["links"]["self"])
if download.status_code >= 400:
log.error("Unable to download 'components.json'")
log.error("Response was {}".format(download.json()))
sys.exit(1)
# component response has checksum as "md5:HEXDIGEST", strip the md5.
if hashlib.md5(download.content).hexdigest() != componentjson["checksum"][4:]:
log.error("Download failed checksum, expecting {expect}, got {got}".format(
expect=componentjson["checksum"][4:],
got=hashlib.md5(download.content).hexdigest()))
sys.exit(1)
componentjson = download.json()
branches = {}
for record in componentjson["components"]:
commit = record["commit"]
component = record["component"]
package = component[component.find("/")+1:].lower()
branches[package] = commit
log.info("Using commit {commit} for component {comp}".format(
commit=commit, comp=component))
return branches


if mode == "install":
# Handle command line arguments.
parser = ArgumentParser(description="""Install firedrake and its dependencies.""",
Expand Down Expand Up @@ -145,6 +211,12 @@ honoured.""",
help="Pointing to external Python packages is usually a user error. Set this option if you know that you want PYTHONPATH set.")
parser.add_argument("--rebuild-script", action="store_true",
help="Only rebuild the firedrake-install script. Use this option if your firedrake-install script is broken and a fix has been released in upstream Firedrake. You will need to specify any other options which you wish to be honoured by your new update script.")
parser.add_argument("--doi", type=str, nargs=1,
help="Install a set of components matching a particular Zenodo DOI. The record should have been created with firedrake-zenodo.")
# Used for testing if Zenodo broke the API
# Tries to resolve to a known DOI and immediately exits.
parser.add_argument("--test-doi-resolution", action="store_true",
help=argparse.SUPPRESS)
parser.add_argument("--package-branch", type=str, nargs=2, action="append", metavar=("PACKAGE", "BRANCH"),
help="Specify which branch of a package to use. This takes two arguments, the package name and the branch.")
parser.add_argument("--verbose", "-v", action="store_true", help="Produce more verbose debugging output.")
Expand Down Expand Up @@ -173,7 +245,27 @@ honoured.""",
args = parser.parse_args()

if args.package_branch:
branches = {package: branch for package, branch in args.package_branch}
branches = {package.lower(): branch for package, branch in args.package_branch}
if args.test_doi_resolution:
actual = resolve_doi_branches("10.5281/zenodo.1322546")
expect = {'coffee': '87e50785d3a05b111f5423a66d461cd44cc4bdc9',
'finat': 'aa74fd499304c8363a4520555fd62ef21e8e5e1f',
'fiat': '184601a46c24fb5cbf8fd7961d22b16dd26890e7',
'firedrake': '6a30b64da01eb587dcc0e04e8e6b84fe4839bdb7',
'petsc': '413f72f04f5cb0a010c85e03ed029573ff6d4c63',
'petsc4py': 'ac2690070a80211dfdbab04634bdb3496c14ca0a',
'tsfc': 'fe9973eaacaa205fd491cd1cc9b3743b93a3d076',
'ufl': 'c5eb7fbe89c1091132479c081e6fa9c182191dcc',
'pyop2': '741a21ba9a62cb67c0aa300a2e199436ea8cb61c'}
if actual != expect:
log.error("Unable to resolve DOI correctly.")
log.error("You'll need to figure out how Zenodo have changed their API.")
sys.exit(1)
else:
log.info("DOI resolution test passed.")
sys.exit(0)
if args.doi:
branches = resolve_doi_branches(args.doi[0])

args.prefix = False # Disabled as untested
args.packages = args.packages or []
Expand Down Expand Up @@ -249,23 +341,6 @@ where shortname is one of the names given below:
print(help_string)
sys.exit(0)

# Set up logging
# Log to file at DEBUG level
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)-6s %(message)s',
filename='firedrake-%s.log' % mode,
filemode='w')
# Log to console at INFO level
console = logging.StreamHandler()
console.setLevel(logging.INFO)
formatter = logging.Formatter('%(message)s')
console.setFormatter(formatter)
logging.getLogger().addHandler(console)

log = logging.getLogger()

log.info("Running %s" % " ".join(sys.argv))


@atexit.register
def print_log_location():
Expand Down Expand Up @@ -436,8 +511,7 @@ def git_clone(url):
log.info("Using existing petsc installation\n")
return name
log.info("Cloning %s\n" % name)
if name in branches:
branch = branches[name]
branch = branches.get(name.lower(), branch)
try:
if options["disable_ssh"]:
raise InstallError("Skipping ssh clone because --disable-ssh")
Expand Down

0 comments on commit 1951865

Please sign in to comment.