Merge branch 'master' into add-release-diff-action

.github/workflows/build-and-test.yml

@@ -6,6 +6,7 @@ on:
       - master
     types:
       - opened
+      - synchronize
     paths-ignore:
       - '**.md'
       - 'CITATION.cff'

.github/workflows/constraints.yml

@@ -18,6 +18,7 @@ jobs:
         run: sh update-constraints.sh
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
+        if: ${{ success() }}
         with:
           commit-message: Update constraints.txt
           title: 'Update constraints.txt'

CHANGELOG.md

@@ -1,19 +1,54 @@
-# v1.3.2 (unpublished, target date 31 July 2022)
+# v1.5 (unpublished)
+
+A full record of all planned changes can be seen [here](https://github.com/INCATools/ontology-development-kit/milestone/7).
+
+- New [ROBOT version 1.9.5](https://github.com/ontodev/robot/releases/tag/v1.9.5)
+- All processes within the ODK container now runs under the identify of an unprivileged user by default. This fixes the issue of generated files being owned by the superuser, when the Docker daemon itself runs as the superuser (as is the case by default on most GNU/Linux systems). If you have a workflow that requires being run as a superuser (for example, if you need to install extra Debian/Ubuntu packages via `apt-get`), set the environment variable `ODK_USER_ID` to 0.
+
+## Makefile workflows
+
+- Added a `test_fast` goal that runs all ontology QC checks without refreshing imports or rebuilding components.
+
+# v1.4
+
+A full record of all changes can be seen [here](https://github.com/INCATools/ontology-development-kit/milestone/6?closed=1).
+
+## New and updated tooling
+
+- New [ROBOT Version 1.9.3](https://github.com/ontodev/robot/releases/tag/v1.9.3)
+- A lot of [updated python tools](https://github.com/INCATools/ontology-development-kit/blob/master/constraints.txt), including [OAK (0.1.71)](https://github.com/INCATools/ontology-access-kit), [SSSOM tools](https://github.com/mapping-commons/sssom-py) (0.3.22), [LinkML](https://linkml.io/) (1.4.4) and [curies](https://github.com/cthoyt/curies) (0.4.2).
+- `gh` is now installed in ODK, which means that GitHub workflows can be run directly through ODK. For example, try out the new `public_release` workflow which automatically creates a GitHub release for you.
+- A full list of all available python tools and there vesions is available [here](https://github.com/INCATools/ontology-development-kit/blob/master/constraints.txt)
 
 ## New configuration options 
 
 - Making the `uribase` configurable. You can now set the URI base in your `myont-odk.yaml` file to something different from http://purl.obolibrary.org/obo/, which enables developers from outside OBO to use ODK. Note that there is no guarantee that you can export your ontology to the OBO file format of you customise your `baseuri` this way!
+- `import_component_format`: You can now configure the format in which your imports and components are serialised. For example, rather than owl (RDFXML), which used to be the default, you can set this option to ofn. For a complete reference see https://robot.obolibrary.org/convert.
+- `mirror_type`: You can define the type of the mirror for each import. Supported: base, custom, no_mirror.
+- `release_use_reasoner`: If false, no reasoning is performed for generating release files. This is only relevant for building application ontologies, where all components are already fully classified.
+- `sparql_test_on`: You can define the list of input files to run the custom SPARQL queries. Supported: edit, and any release artefact, e.g myont-base.owl ([example](https://github.com/INCATools/ontology-development-kit/blob/master/tests/test-sparql-report.yaml)).
+- `use_edit_file_imports`: If TRUE, use whatever imports you have in the edit file to create the release (default). If FALSE,  components (and imports) are merged into the release independent of whether they are mentioned in the edit file or not. This can help for example in cases where import modules are so large, they cannot be checked into version control.
+- The `ci` option now takes `gitlab-ci` as a value, which sets your repo up with basic Continuous Integration Testing for GitLab.
 
 ## Makefile workflows
 
 - Adding new `make reason_test` command ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/645))
 - Adding intermediate artefact `$(TMPDIR)/$(ONT)-preprocess.owl` to the release workflow which enable the centralisation of preprocessing in a single make step. Basically, rather than creating release artefacts directly from the editors file (e.g. bfo-edit.owl), we add intermediate step from which all releases are derived. This intermediate can then be customised by the user ([pull](https://github.com/INCATools/ontology-development-kit/pull/639), [issue](https://github.com/INCATools/ontology-development-kit/issues/544))
+- Introduces an experimental new release called "base-plus", which includes the inferred and non-redundant classification of the ontology. This is highly experimental and may be removed in a future release of ODK. The new goal is much more rigorous in removing axioms from other ontologies as well. The old base file can now be exported as an  "editors release" instead ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/646)).
+- Introduces the option of skipping the use of reasoner during the release process (important for application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/644))
+- Introduces a new mode that enable the use of ODK entirely without owl:imports in the edit file (this is great in case we want to use the ODK workflows but not check in any files, imports or components, into version control, like huge application ontologies), ([pull](https://github.com/INCATools/ontology-development-kit/pull/643), [issue](https://github.com/INCATools/ontology-development-kit/issues/629)).
+- Adds a feature to directly support ROBOT templates in components ([example](https://github.com/INCATools/ontology-development-kit/blob/master/tests/test-templates.yaml)). Templates need to be activates with the project-level `use_templates: TRUE` option, and can then be used to define components, like in the example.
+- Adds an option to do a `public_release` on Github which creates a GitHub release, tags it, and uploads release artefacts.
+- Adds a `release_diff` workflow which creates a simple markdown report between the latest release and the current one.
+- Adds a feature to directly support SSSOM mapping files, both in components and as standalone ([example](https://github.com/INCATools/ontology-development-kit/blob/master/tests/test-sssom.yaml)). This allows to curate, for example, SSSOM mapping files in tables and them release them as part of the general release process. It also allows extracting mappings from inside the Ontology automatically using the SSSOM toolkit.
 - Changes in the `src/ontology/run.sh` wrapper script:
   - [It is now possible](https://github.com/INCATools/ontology-development-kit/pull/640) to execute the Docker image through [Singularity](https://apptainer.org).
   - The `IMAGE` variable, which can used to specify an alternative ODK image, [has been renamed](https://github.com/INCATools/ontology-development-kit/pull/655) to `ODK_IMAGE`.
   - A new variable `ODK_TAG` has been introduced, allowing to specify an alternative tag (default is `latest`). A tag may also be specified directly as part of the `ODK_IMAGE` variable (as in `ODK_IMAGE=odkfull:v1.3.1`).
   - A new variable `ODK_BINDS` has been introduced, allowing to specify extra bindings between a directory on the host computer and a directory inside the Docker container.
   - Variables used by the `run.sh` script can now be set in a `src/ontology/run.sh.conf` file, which will be sourced by the wrapper script.
+  - With the config file option `use_env_file_docker`, if true, you can pass your local environment to the docker container. It is _strongly advised_ to add `run.sh.env` to `.gitignore` before using this feature. Committing your environment to git may reveal passcodes and other private information!
+
 
 # v1.3.1
 
@@ -246,7 +281,7 @@ custom_makefile_header: |
 # v1.2.24 (6 November 2020)
 - Updated ROBOT to new version 1.7.1
 - Added the (highly experimental) ability to ODK to run OBO dashboard (see [instructions and examples](https://github.com/INCATools/ontology-development-kit#running-obo-dashboard-with-odk)).
-- Added more python packages to ODK, see [requirements.txt](requirements.txt).
+- Added more python packages to ODK, see [requirements.txt](requirements.txt.full).
 - Added a new set of configurations for ROBOT report. WARNING:
 `report_fail_on` option is now deprecated in favour of a new block of options:
 
@@ -387,7 +422,7 @@ import_group:
 
  * project.yaml files can be passed instead of command line settings
  * Docker container can now be executed anywhere, no checkout required
- * Added [README-developers.md](README-developers.md) for ODK developers
+ * Added [README-developers.md](CONTRIBUTING.md) for ODK developers
  * Docker container tracking ROBOT v1.2
  * Previously multiple dependencies: `-d ont1 ont2 ... ontN`, now `-d ont1 -id ont2 ... -d ontN`
  * New command line option `--source` to pass in an existing ontolgy-edit file

CONTRIBUTING.md

@@ -122,6 +122,14 @@ preserving protected files. TBD how to determine protected
 files. Obviously the edit file should not be touched. Could use git
 log to determine if any modifications have been made?
 
+## Setting up a new machine for ODK development
+
+In order to build and publish ODK, you need the following:
+
+1. `docker` installed
+2. have `git` installed
+3. have a git user.name and user.email set
+
 ## General SOP for ODK release and publication
 
 * Put the `master` branch in the state we want for release (i.e. merge any approved PR that we want included in that release, etc.).
@@ -257,6 +265,20 @@ The examples folder serves for both unit test and documentation purposes.
 
 TODO
 
+## Pull request rules
+
+1. One PR per feature.
+2. Each PR must link to one or more existing issues.
+3. There should be one commit per logical change. This is important so we can be more effective at cherry picking for patch releases.
+4. Every commit should have an appropriate title and description.
+
+## Major minor system
+
+- Once a release is done, we work on the master branch towards the next major release
+- The minor release steward cherry picks certain kinds of changes for minor releases, including
+   - ROBOT updates
+   - Python tool version updates
+   - Critical bug fixes 
 
 ## Adding new programs or Python modules to the ODK
 
@@ -284,9 +306,8 @@ Java programs available as pre-built jars can be installed by adding new
 `RUN` commands at the end of either the main Dockerfile (for `odkfull`)
 or the Dockerfile for `odklite`.
 
-If the component needs to be built from source, do so in [the Dockerfile
-for odkbuild](docker/build/Dockerfile), and install the compiled file(s)
-in either the `/staging/full` tree or the `/staging/lite` tree, for
+If the component needs to be built from source, do so in [the Dockerfile for odkbuild](docker/builder/Dockerfile), 
+and install the compiled file(s) in either the `/staging/full` tree or the `/staging/lite` tree, for
 inclusion in `odkfull` or `odklite` respectively.
 
 If the component is a Python package, adds it to the `requirements.txt`

Dockerfile

@@ -9,6 +9,10 @@ ENV PATH "/tools/apache-jena/bin:/tools/sparqlprog/bin:$PATH"
 ARG ODK_VERSION 0.0.0
 ENV ODK_VERSION $ODK_VERSION
 
+# Software versions
+ENV JENA_VERSION=4.9.0
+ENV SSSOM_JAVA_VERSION=0.7.3
+
 # Avoid repeated downloads of script dependencies by mounting the local coursier cache:
 # docker run -v $HOME/.coursier/cache/v1:/tools/.coursier-cache ...
 ENV COURSIER_CACHE "/tools/.coursier-cache"
@@ -30,6 +34,7 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-i
     dos2unix \
     sqlite3 \
     libjson-perl \
+    libbusiness-isbn-perl \
     pkg-config \
     xlsx2csv \
     gh \
@@ -51,9 +56,9 @@ RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends
 COPY --from=obolibrary/odkbuild:latest /staging/full /
 
 # Install Konclude.
-# On x86_64, we get it from a pre-built release; on arm64, we
-# have built it in the builder image, now we need to install
-# the run-time dependencies (Qt5).
+# On x86_64, we get it from a pre-built release from upstream; on arm64,
+# we use a custom pre-built binary to which we just need to add the
+# run-time dependencies (the binary is not statically linked).
 ARG TARGETARCH
 RUN test "x$TARGETARCH" = xamd64 && ( \
         wget -nv https://github.com/konclude/Konclude/releases/download/v0.7.0-1138/Konclude-v0.7.0-1138-Linux-x64-GCC-Static-Qt5.12.10.zip \
@@ -64,12 +69,18 @@ RUN test "x$TARGETARCH" = xamd64 && ( \
         rm Konclude.zip \
     ) || ( \
         DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-install-recommends \
-            libqt5xml5 libqt5network5 libqt5concurrent5 \
+            libqt5xml5 libqt5network5 libqt5concurrent5 && \
+        wget -nv https://incenp.org/files/softs/konclude/0.7/Konclude-v0.7.0-1138-Linux-arm64-GCC.zip \
+            -O /tools/Konclude.zip && \
+        unzip Konclude.zip && \
+        mv Konclude-v0.7.0-1138-Linux-arm64-GCC/Binaries/Konclude /tools/Konclude && \
+        rm -rf Konclude-v0.7.0-1138-Linux-arm64-GCC && \
+        rm Konclude.zip \
     )
 
 # Install Jena.
-RUN wget -nv http://archive.apache.org/dist/jena/binaries/apache-jena-4.6.1.tar.gz -O- | tar xzC /tools && \
-    mv /tools/apache-jena-4.6.1 /tools/apache-jena
+RUN wget -nv http://archive.apache.org/dist/jena/binaries/apache-jena-$JENA_VERSION.tar.gz -O- | tar xzC /tools && \
+    mv /tools/apache-jena-$JENA_VERSION /tools/apache-jena
 
 # Install SPARQLProg.
 RUN swipl -g "pack_install(sparqlprog, [interactive(false)])" -g halt && \
@@ -96,3 +107,9 @@ RUN wget -nv https://github.com/balhoff/relation-graph/releases/download/v$RG/re
 && tar -zxvf relation-graph-cli-$RG.tgz \
 && mv relation-graph-cli-$RG /tools/relation-graph \
 && chmod +x /tools/relation-graph
+
+# Install SSSOM-Java
+RUN wget -nv -O /tools/sssom-cli https://github.com/gouttegd/sssom-java/releases/download/sssom-java-$SSSOM_JAVA_VERSION/sssom-cli && \
+    chmod +x /tools/sssom-cli && \
+    mkdir -p /tools/robot-plugins && \
+    wget -nv -O /tools/robot-plugins/sssom.jar https://github.com/gouttegd/sssom-java/releases/download/sssom-java-$SSSOM_JAVA_VERSION/sssom-robot-plugin-$SSSOM_JAVA_VERSION.jar

Makefile

@@ -11,7 +11,7 @@ EMAIL_ARGS=
 
 CACHE=
 
-ARCH=$(shell uname -m | sed 's/x86_64/amd64/')
+ARCH=linux/$(shell uname -m | sed 's/x86_64/amd64/')
 PLATFORMS=linux/amd64,linux/arm64
 
 .PHONY: .FORCE
@@ -54,16 +54,14 @@ test: $(TEST_FILES) custom_tests
 tests/*.yaml: .FORCE
 	$(CMD) -c -C $@
 
-schema/project-schema.json:
-	./odk/odk.py dump-schema > $@
 
-docs/project-schema.md: schema/project-schema.json
-	python odk/schema_documentation.py
-
-docs: docs/project-schema.md
+.PHONY: docs
+docs:
+	@ODK_IMAGE=odklite ./odk.sh ./odk/odk.py dump-schema > schema/project-schema.json
+	@ODK_IMAGE=odklite ./odk.sh python ./odk/schema_documentation.py
 
 # Building docker image
-VERSION = "v1.3.2"
+VERSION = "v1.5"
 IM=obolibrary/odkfull
 IMLITE=obolibrary/odklite
 ROB=obolibrary/robot
@@ -180,3 +178,8 @@ dev-test-publish:
 	docker buildx rm multiarch
 	docker buildx create --name multiarch --driver docker-container --use
 	$(MAKE) tests publish-multiarch-dev
+
+dev-test-publish-no-rm:
+	git pull
+	docker buildx create --name multiarch --driver docker-container --use
+	$(MAKE) tests publish-multiarch-dev