diff --git a/.gitignore b/.gitignore index 0a0b986..6553790 100644 --- a/.gitignore +++ b/.gitignore @@ -191,6 +191,7 @@ prototyping/ cssr/ presentation/ paper/ +playground/ project/ .vscode/ diff --git a/MANIFEST.in b/MANIFEST.in index 30ada54..65ad799 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,7 +7,5 @@ include src/pyopmnearwell/templates/co2store/* include src/pyopmnearwell/templates/common/* include src/pyopmnearwell/templates/h2store/* include src/pyopmnearwell/templates/saltprec/* -include src/pyopmnearwell/templates/standardwell/* -include src/pyopmnearwell/templates/standardwell_impl/* include src/pyopmnearwell/utils/* include src/pyopmnearwell/visualization/* \ No newline at end of file diff --git a/README.md b/README.md index 284a6d8..172342d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Build Status](https://github.com/cssr-tools/pyopmnearwell/actions/workflows/CI.yml/badge.svg)](https://github.com/cssr-tools/pyopmnearwell/actions/workflows/CI.yml) - + [![Code style](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/ambv/black) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) [![DOI](https://zenodo.org/badge/662625461.svg)](https://zenodo.org/doi/10.5281/zenodo.10266790) @@ -13,9 +13,16 @@ using the [_OPM-Flow_](https://opm-project.org/?page_id=19) simulator. ## Installation You will first need to install -* Flow (https://opm-project.org, Release 2023.10 or current master branches) +* OPM Flow (https://opm-project.org, Release 2024.04 or current master branches) -You can install the requirements in a virtual environment with the following commands: +To install the _pyopmnearwell_ executable in an existing Python environment: + +```bash +pip install git+https://github.com/cssr-tools/pyopmnearwell.git +``` + +If you are interested in modifying the source code, then you can clone the repository and +install the Python requirements in a virtual environment with the following commands: ```bash # Clone the repo @@ -28,16 +35,14 @@ python3 -m venv vpyopmnearwell source vpyopmnearwell/bin/activate # Upgrade pip, setuptools, and wheel pip install --upgrade pip setuptools wheel -# Install the pyopmnearwell package (in editable mode for contributions/modifications; otherwise, pip install .) +# Install the pyopmnearwell package pip install -e . # For contributions/testing/linting, install the dev-requirements pip install -r dev-requirements.txt ``` -See the [_CI.yml_](https://github.com/cssr-tools/pyopmnearwell/blob/main/.github/workflows/CI.yml) script -for installation of OPM Flow (binary packages) and the pyopmnearwell package. If you are a Linux user (including the windows subsystem for Linux), then you could try to build Flow from the master branches with mpi support, by running the script `./build_opm-flow_mpi.bash`, which in turn should build flow in the folder ./build/opm-simulators/bin/flow. - -For macOS users with the latest chips (M1/M2, guessing also M3?), the ecl, resdata, and opm Python packages are not available via pip install. Then before installation, remove them from the `requirements.txt`, then proceed with the Python requirements installation, install the OPM Flow dependencies (using macports or brew), and once inside the vpyopmnearwell Python environment, run the `./build_opm-flow_macOS.bash`, and deactivate and activate the virtual environment (this script builds OPM Flow as well as the opm Python package, and it exports the required PYTHONPATH). +See the [_installation_](https://cssr-tools.github.io/pyopmnearwell/examples.html) for further details on building OPM Flow from the master branches +in Linux, Windows, and macOS. ## Running pyopmnearwell You can run _pyopmnearwell_ as a single command line: @@ -47,11 +52,10 @@ pyopmnearwell -i some_input.txt -o some_output_folder Run `pyopmnearwell --help` to see all possible command line argument options. Inside the `some_input.txt` file you provide the path to the flow executable and simulation parameters. See the .txt files in the `examples/`, -`tests/geometries/`, and `tests/models/` folders. For macOS users, then use the flag -`-p opm` for plotting (resdata is the default one). +`tests/geometries/`, and `tests/models/` folders. ## Getting started -See the [_documentation_](https://cssr-tools.github.io/pyopmnearwell/introduction.html). +See the [_examples_](https://cssr-tools.github.io/pyopmnearwell/examples.html) in the [_documentation_](https://cssr-tools.github.io/pyopmnearwell/introduction.html). ## About pyopmnearwell The pyopmnearwell package is being funded by the [_HPC Simulation Software for the Gigatonne Storage Challenge project_](https://www.norceresearch.no/en/projects/hpc-simulation-software-for-the-gigatonne-storage-challenge) [project number 622059] and [_Center for Sustainable Subsurface Resources (CSSR)_](https://cssr.no) diff --git a/build_opm-flow_macOS.bash b/build_opm-flow_macOS.bash deleted file mode 100755 index 1b048a4..0000000 --- a/build_opm-flow_macOS.bash +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: 2023 NORCE -# SPDX-License-Identifier: MIT - -CURRENT_DIRECTORY="$PWD" - -# Dune modules -for module in common geometry grid istl -do git clone https://gitlab.dune-project.org/core/dune-$module.git --branch v2.9.0 -done -for module in common geometry grid istl -do ./dune-common/bin/dunecontrol --only=dune-$module cmake -DCMAKE_DISABLE_FIND_PACKAGE_MPI=1 - ./dune-common/bin/dunecontrol --only=dune-$module make -j5 -done - -# OPM modules -for repo in common grid models simulators -do - git clone https://github.com/OPM/opm-$repo.git -done - -source vpyopmnearwell/bin/activate - -mkdir build - -for repo in common grid models -do - mkdir build/opm-$repo - cd build/opm-$repo - cmake -DPYTHON_EXECUTABLE=$(which python) -DUSE_MPI=0 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo - make -j5 - cd ../.. -done - -mkdir build/opm-simulators -cd build/opm-simulators -cmake -DUSE_MPI=0 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators -make -j5 flow -cd ../.. - -echo "export PYTHONPATH=\$PYTHONPATH:$CURRENT_DIRECTORY/build/opm-common/build/python" >> $CURRENT_DIRECTORY/vpyopmnearwell/bin/activate \ No newline at end of file diff --git a/build_opm-flow_mpi.bash b/build_opm-flow_mpi.bash deleted file mode 100644 index 5178b68..0000000 --- a/build_opm-flow_mpi.bash +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: 2023 NORCE -# SPDX-License-Identifier: MIT - -CURRENT_DIRECTORY="$PWD" - -# OPM modules -for repo in common grid models simulators -do - git clone https://github.com/OPM/opm-$repo.git -done - -mkdir build - -for repo in common grid models -do - mkdir build/opm-$repo - cd build/opm-$repo - cmake -DUSE_MPI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo - make -j5 - cd ../.. -done - -mkdir build/opm-simulators -cd build/opm-simulators -cmake -DUSE_MPI=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators -make -j5 flow -cd ../.. \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index de0a4ea..e6a74d0 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ black -pylint +numpydoc mypy +pylint pytest-cov sphinx -sphinx-rtd-theme -pytest \ No newline at end of file +sphinx-rtd-theme \ No newline at end of file diff --git a/docs/_images/contents.png b/docs/_images/contents.png index 7b84428..19719b0 100644 Binary files a/docs/_images/contents.png and b/docs/_images/contents.png differ diff --git a/docs/_images/plopm.png b/docs/_images/plopm.png new file mode 100644 index 0000000..b7b084f Binary files /dev/null and b/docs/_images/plopm.png differ diff --git a/docs/_images/pycopm.gif b/docs/_images/pycopm.gif new file mode 100644 index 0000000..40d2df9 Binary files /dev/null and b/docs/_images/pycopm.gif differ diff --git a/docs/_images/pyopmspe11.gif b/docs/_images/pyopmspe11.gif index 1d22e7d..e0a88a5 100644 Binary files a/docs/_images/pyopmspe11.gif and b/docs/_images/pyopmspe11.gif differ diff --git a/docs/_sources/api.rst.txt b/docs/_sources/api.rst.txt index c4b7414..afe0b0c 100644 --- a/docs/_sources/api.rst.txt +++ b/docs/_sources/api.rst.txt @@ -7,8 +7,8 @@ The scripts in the utils folder process the input configuration file, creates th write the input files by using the scripts in the templates folder, and execute OPM Flow. The scripts in the visualization folder contains files for the postprocessing of the results to generate figures (.png) to show comparisons between the different runs/parts of the geological model such as well injectivity (WI) and -2D spatial maps for the last time step simulated. The ml folder is under development, see the files in the -'examples/cemracs2023/peaceman_well-model_nn' if you are curious. +2D spatial maps for the last time step simulated. The ml folder constains scripts used in +`this repository `_. .. figure:: figs/contents.png diff --git a/docs/_sources/examples.rst.txt b/docs/_sources/examples.rst.txt index 1f81d1c..ebf390d 100644 --- a/docs/_sources/examples.rst.txt +++ b/docs/_sources/examples.rst.txt @@ -75,3 +75,11 @@ The following are some of the plots created by the **pyopmnearwell** executable: Permeability (top), CO2 injection schedule (middle), and saturation values over time on the cells along the well location at three different locations (bottom). + +Publications +------------ +For the simulation results published in `this paper `_ +about the impact of intermittency on salt precipitation during CO2 injection, see/run +`these configuration files `_. + +For a study where **pyopmnearwell** is used to generated a machine-learned near-well model, `click here `_. \ No newline at end of file diff --git a/docs/_sources/index.rst.txt b/docs/_sources/index.rst.txt index 0a3582f..3637eaf 100644 --- a/docs/_sources/index.rst.txt +++ b/docs/_sources/index.rst.txt @@ -7,6 +7,7 @@ Welcome to pyopmnearwell's documentation! :maxdepth: 4 introduction + installation configuration_file examples api diff --git a/docs/_sources/installation.rst.txt b/docs/_sources/installation.rst.txt new file mode 100644 index 0000000..0f592a7 --- /dev/null +++ b/docs/_sources/installation.rst.txt @@ -0,0 +1,128 @@ +============ +Installation +============ + +Python package +-------------- + +To install the **pyopmnearwell** executable in an existing Python environment: + +.. code-block:: bash + + pip install git+https://github.com/cssr-tools/pyopmnearwell.git + +If you are interested in modifying the source code, then you can clone the repository and +install the Python requirements in a virtual environment with the following commands: + +.. code-block:: console + + # Clone the repo + git clone https://github.com/cssr-tools/pyopmnearwell.git + # Get inside the folder + cd pyopmnearwell + # Create virtual environment + python3 -m venv vpyopmnearwell + # Activate virtual environment + source vpyopmnearwell/bin/activate + # Upgrade pip, setuptools, and wheel + pip install --upgrade pip setuptools wheel + # Install the pyopmnearwell package + pip install -e . + # For contributions/testing/linting, install the dev-requirements + pip install -r dev-requirements.txt + +OPM Flow +-------- +You also need to install: + +* OPM Flow (https://opm-project.org, Release 2024.04 or current master branches) + +.. tip:: + + See the `CI.yml `_ script + for installation of OPM Flow (binary packages) and the pyopmnearwell package in Linux. + +Source build in Linux/Windows ++++++++++++++++++++++++++++++ +If you are a Linux user (including the Windows subsystem for Linux), then you could try to build Flow (after installing the `prerequisites `_) from the master branches with mpi support by running +in the terminal the following lines (which in turn should build flow in the folder ./build/opm-simulators/bin/flow.): + +.. code-block:: console + + CURRENT_DIRECTORY="$PWD" + + for repo in common grid models simulators + do + git clone https://github.com/OPM/opm-$repo.git + done + + mkdir build + + for repo in common grid models + do + mkdir build/opm-$repo + cd build/opm-$repo + cmake -DUSE_MPI=1 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo + make -j5 opm$repo + cd ../.. + done + + mkdir build/opm-simulators + cd build/opm-simulators + cmake -DUSE_MPI=1 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators + make -j5 flow + cd ../.. + + +.. tip:: + + You can create a .sh file (e.g., build_opm_mpi.sh), copy the previous lines, and run in the terminal **source build_opm_mpi.sh** + +Regarding the plotting functionality, it is possible to use the opm python package instead of resdata (e.g., it seems the opm Python package +is faster than resdata to read large simulation files). To use opm, you first need to install it, by executing in the terminal **pip install opm**. + +Source build in macOS ++++++++++++++++++++++ +For macOS, there are no available binary packages, so OPM Flow needs to be built from source, in addition to the dune libraries and the opm Python +package (see the `prerequisites `_, which can be installed using macports or brew). This can be achieved by the following lines: + +.. code-block:: console + + CURRENT_DIRECTORY="$PWD" + + for module in common geometry grid istl + do git clone https://gitlab.dune-project.org/core/dune-$module.git --branch v2.9.1 + done + for module in common geometry grid istl + do ./dune-common/bin/dunecontrol --only=dune-$module cmake -DCMAKE_DISABLE_FIND_PACKAGE_MPI=1 + ./dune-common/bin/dunecontrol --only=dune-$module make -j5 + done + + for repo in common grid models simulators + do + git clone https://github.com/OPM/opm-$repo.git + done + + source vpyopmnearwell/bin/activate + + mkdir build + + for repo in common grid models + do + mkdir build/opm-$repo + cd build/opm-$repo + cmake -DPYTHON_EXECUTABLE=$(which python) -DWITH_NDEBUG=1 -DUSE_MPI=0 -DOPM_ENABLE_PYTHON=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid" $CURRENT_DIRECTORY/opm-$repo + make -j5 opm$repo + cd ../.. + done + + mkdir build/opm-simulators + cd build/opm-simulators + cmake -DUSE_MPI=0 -DWITH_NDEBUG=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_PREFIX_PATH="$CURRENT_DIRECTORY/dune-common/build-cmake;$CURRENT_DIRECTORY/dune-grid/build-cmake;$CURRENT_DIRECTORY/dune-geometry/build-cmake;$CURRENT_DIRECTORY/dune-istl/build-cmake;$CURRENT_DIRECTORY/build/opm-common;$CURRENT_DIRECTORY/build/opm-grid;$CURRENT_DIRECTORY/build/opm-models" $CURRENT_DIRECTORY/opm-simulators + make -j5 flow + cd ../.. + + echo "export PYTHONPATH=\$PYTHONPATH:$CURRENT_DIRECTORY/build/opm-common/build/python" >> $CURRENT_DIRECTORY/vpyopmnearwell/bin/activate + + +This builds OPM Flow as well as the opm Python package, and it exports the required PYTHONPATH. Then after execution, deactivate and activate the Python virtual environment. \ No newline at end of file diff --git a/docs/_sources/introduction.rst.txt b/docs/_sources/introduction.rst.txt index f0f0f15..5f3f7f0 100644 --- a/docs/_sources/introduction.rst.txt +++ b/docs/_sources/introduction.rst.txt @@ -7,7 +7,7 @@ Introduction This documentation describes the content of the **pyopmnearwell** package. The numerical studies are performed using the -`Flow `_ simulator. +`OPM Flow `_ simulator. Concept ------- @@ -44,14 +44,6 @@ where - \-s, \-scale: Scale for the x axis in the figures: 'normal' or 'log' ('normal' by default). - \-m, \-model: Simulated model (5th row in the configuration file). This is used for the plotting compare method (it gets overwritten by the configuration file) ('co2store' by default). -Installation ------------- - -See the `Github page `_. - -.. tip:: - Check the `CI.yml `_ file. - .. warning:: The H2CH4 template in the h2store model folder is under development and it is based on an input deck available in `opm-tests `_. In addition, the templates diff --git a/docs/_sources/pyopmnearwell.ml.analysis.rst.txt b/docs/_sources/pyopmnearwell.ml.analysis.rst.txt new file mode 100644 index 0000000..5683fff --- /dev/null +++ b/docs/_sources/pyopmnearwell.ml.analysis.rst.txt @@ -0,0 +1,8 @@ +pyopmnearwell.ml.analysis module +================================ + +.. automodule:: pyopmnearwell.ml.analysis + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/_sources/pyopmnearwell.ml.data.rst.txt b/docs/_sources/pyopmnearwell.ml.data.rst.txt deleted file mode 100644 index 5fd03ba..0000000 --- a/docs/_sources/pyopmnearwell.ml.data.rst.txt +++ /dev/null @@ -1,8 +0,0 @@ -pyopmnearwell.ml.data module -============================ - -.. automodule:: pyopmnearwell.ml.data - :members: - :undoc-members: - :show-inheritance: - :private-members: diff --git a/docs/_sources/pyopmnearwell.ml.resdata_dataset.rst.txt b/docs/_sources/pyopmnearwell.ml.resdata_dataset.rst.txt new file mode 100644 index 0000000..ce454ef --- /dev/null +++ b/docs/_sources/pyopmnearwell.ml.resdata_dataset.rst.txt @@ -0,0 +1,8 @@ +pyopmnearwell.ml.resdata\_dataset module +======================================== + +.. automodule:: pyopmnearwell.ml.resdata_dataset + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/_sources/pyopmnearwell.ml.rst.txt b/docs/_sources/pyopmnearwell.ml.rst.txt index bc794e3..76c3106 100644 --- a/docs/_sources/pyopmnearwell.ml.rst.txt +++ b/docs/_sources/pyopmnearwell.ml.rst.txt @@ -7,12 +7,15 @@ Submodules .. toctree:: :maxdepth: 4 - pyopmnearwell.ml.data + pyopmnearwell.ml.analysis pyopmnearwell.ml.ensemble pyopmnearwell.ml.integration pyopmnearwell.ml.kerasify pyopmnearwell.ml.nn + pyopmnearwell.ml.resdata_dataset pyopmnearwell.ml.scaler_layers + pyopmnearwell.ml.upscale + pyopmnearwell.ml.utils Module contents --------------- diff --git a/docs/_sources/pyopmnearwell.ml.upscale.rst.txt b/docs/_sources/pyopmnearwell.ml.upscale.rst.txt new file mode 100644 index 0000000..62141e1 --- /dev/null +++ b/docs/_sources/pyopmnearwell.ml.upscale.rst.txt @@ -0,0 +1,8 @@ +pyopmnearwell.ml.upscale module +=============================== + +.. automodule:: pyopmnearwell.ml.upscale + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/_sources/pyopmnearwell.ml.utils.rst.txt b/docs/_sources/pyopmnearwell.ml.utils.rst.txt new file mode 100644 index 0000000..168e344 --- /dev/null +++ b/docs/_sources/pyopmnearwell.ml.utils.rst.txt @@ -0,0 +1,8 @@ +pyopmnearwell.ml.utils module +============================= + +.. automodule:: pyopmnearwell.ml.utils + :members: + :undoc-members: + :show-inheritance: + :private-members: diff --git a/docs/_sources/related.rst.txt b/docs/_sources/related.rst.txt index fc60d50..c7f98e9 100644 --- a/docs/_sources/related.rst.txt +++ b/docs/_sources/related.rst.txt @@ -14,6 +14,23 @@ pyopmspe11 `A Python framework using OPM Flow for the CSP SPE11 benchmark project `_. +****** +pycopm +****** + +.. image:: ./figs/pycopm.gif + :scale: 60% + +`Simplified and flexible framework for coarsening geological models `_. + +***** +plopm +***** + +.. image:: ./figs/plopm.png + +`Quick generation of PNG figures from a simulation model given any 2D slide `_. + ******** expreccs ******** diff --git a/docs/_static/basic.css b/docs/_static/basic.css index 30fee9d..f316efc 100644 --- a/docs/_static/basic.css +++ b/docs/_static/basic.css @@ -4,7 +4,7 @@ * * Sphinx stylesheet -- basic theme. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/docs/_static/doctools.js b/docs/_static/doctools.js index d06a71d..4d67807 100644 --- a/docs/_static/doctools.js +++ b/docs/_static/doctools.js @@ -4,7 +4,7 @@ * * Base JavaScript utilities for all Sphinx HTML documentation. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ diff --git a/docs/_static/language_data.js b/docs/_static/language_data.js index 250f566..367b8ed 100644 --- a/docs/_static/language_data.js +++ b/docs/_static/language_data.js @@ -5,7 +5,7 @@ * This script contains the language-specific data used by searchtools.js, * namely the list of stopwords, stemmer, scorer and splitter. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @@ -13,7 +13,7 @@ var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"]; -/* Non-minified version is copied as a separate JS file, is available */ +/* Non-minified version is copied as a separate JS file, if available */ /** * Porter Stemmer diff --git a/docs/_static/searchtools.js b/docs/_static/searchtools.js index 7918c3f..b08d58c 100644 --- a/docs/_static/searchtools.js +++ b/docs/_static/searchtools.js @@ -4,7 +4,7 @@ * * Sphinx JavaScript utilities for the full-text search. * - * :copyright: Copyright 2007-2023 by the Sphinx team, see AUTHORS. + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. * :license: BSD, see LICENSE for details. * */ @@ -99,7 +99,7 @@ const _displayItem = (item, searchTerms, highlightTerms) => { .then((data) => { if (data) listItem.appendChild( - Search.makeSearchSummary(data, searchTerms) + Search.makeSearchSummary(data, searchTerms, anchor) ); // highlight search terms in the summary if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js @@ -116,8 +116,8 @@ const _finishSearch = (resultCount) => { ); else Search.status.innerText = _( - `Search finished, found ${resultCount} page(s) matching the search query.` - ); + "Search finished, found ${resultCount} page(s) matching the search query." + ).replace('${resultCount}', resultCount); }; const _displayNextItem = ( results, @@ -137,6 +137,22 @@ const _displayNextItem = ( // search finished, update title and status message else _finishSearch(resultCount); }; +// Helper function used by query() to order search results. +// Each input is an array of [docname, title, anchor, descr, score, filename]. +// Order the results by score (in opposite order of appearance, since the +// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically. +const _orderResultsByScoreThenName = (a, b) => { + const leftScore = a[4]; + const rightScore = b[4]; + if (leftScore === rightScore) { + // same score: sort alphabetically + const leftTitle = a[1].toLowerCase(); + const rightTitle = b[1].toLowerCase(); + if (leftTitle === rightTitle) return 0; + return leftTitle > rightTitle ? -1 : 1; // inverted is intentional + } + return leftScore > rightScore ? 1 : -1; +}; /** * Default splitQuery function. Can be overridden in ``sphinx.search`` with a @@ -160,13 +176,26 @@ const Search = { _queued_query: null, _pulse_status: -1, - htmlToText: (htmlString) => { + htmlToText: (htmlString, anchor) => { const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html'); - htmlElement.querySelectorAll(".headerlink").forEach((el) => { el.remove() }); + for (const removalQuery of [".headerlink", "script", "style"]) { + htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() }); + } + if (anchor) { + const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`); + if (anchorContent) return anchorContent.textContent; + + console.warn( + `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.` + ); + } + + // if anchor not specified or not found, fall back to main content const docContent = htmlElement.querySelector('[role="main"]'); - if (docContent !== undefined) return docContent.textContent; + if (docContent) return docContent.textContent; + console.warn( - "Content block not found. Sphinx search tries to obtain it via '[role=main]'. Could you check your theme or template." + "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template." ); return ""; }, @@ -239,16 +268,7 @@ const Search = { else Search.deferQuery(query); }, - /** - * execute search (requires search index to be loaded) - */ - query: (query) => { - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const titles = Search._index.titles; - const allTitles = Search._index.alltitles; - const indexEntries = Search._index.indexentries; - + _parseQuery: (query) => { // stem the search terms and add them to the correct list const stemmer = new Stemmer(); const searchTerms = new Set(); @@ -284,21 +304,38 @@ const Search = { // console.info("required: ", [...searchTerms]); // console.info("excluded: ", [...excludedTerms]); - // array of [docname, title, anchor, descr, score, filename] - let results = []; + return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; + }, + + /** + * execute search (requires search index to be loaded) + */ + _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + + // Collect multiple result groups to be sorted separately and then ordered. + // Each is an array of [docname, title, anchor, descr, score, filename]. + const normalResults = []; + const nonMainIndexResults = []; + _removeChildren(document.getElementById("search-progress")); - const queryLower = query.toLowerCase(); + const queryLower = query.toLowerCase().trim(); for (const [title, foundTitles] of Object.entries(allTitles)) { - if (title.toLowerCase().includes(queryLower) && (queryLower.length >= title.length/2)) { + if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) { for (const [file, id] of foundTitles) { - let score = Math.round(100 * queryLower.length / title.length) - results.push([ + const score = Math.round(Scorer.title * queryLower.length / title.length); + const boost = titles[file] === title ? 1 : 0; // add a boost for document titles + normalResults.push([ docNames[file], titles[file] !== title ? `${titles[file]} > ${title}` : title, id !== null ? "#" + id : "", null, - score, + score + boost, filenames[file], ]); } @@ -308,46 +345,47 @@ const Search = { // search for explicit entries in index directives for (const [entry, foundEntries] of Object.entries(indexEntries)) { if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) { - for (const [file, id] of foundEntries) { - let score = Math.round(100 * queryLower.length / entry.length) - results.push([ + for (const [file, id, isMain] of foundEntries) { + const score = Math.round(100 * queryLower.length / entry.length); + const result = [ docNames[file], titles[file], id ? "#" + id : "", null, score, filenames[file], - ]); + ]; + if (isMain) { + normalResults.push(result); + } else { + nonMainIndexResults.push(result); + } } } } // lookup as object objectTerms.forEach((term) => - results.push(...Search.performObjectSearch(term, objectTerms)) + normalResults.push(...Search.performObjectSearch(term, objectTerms)) ); // lookup as search terms in fulltext - results.push(...Search.performTermsSearch(searchTerms, excludedTerms)); + normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms)); // let the scorer override scores with a custom scoring function - if (Scorer.score) results.forEach((item) => (item[4] = Scorer.score(item))); - - // now sort the results by score (in opposite order of appearance, since the - // display function below uses pop() to retrieve items) and then - // alphabetically - results.sort((a, b) => { - const leftScore = a[4]; - const rightScore = b[4]; - if (leftScore === rightScore) { - // same score: sort alphabetically - const leftTitle = a[1].toLowerCase(); - const rightTitle = b[1].toLowerCase(); - if (leftTitle === rightTitle) return 0; - return leftTitle > rightTitle ? -1 : 1; // inverted is intentional - } - return leftScore > rightScore ? 1 : -1; - }); + if (Scorer.score) { + normalResults.forEach((item) => (item[4] = Scorer.score(item))); + nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item))); + } + + // Sort each group of results by score and then alphabetically by name. + normalResults.sort(_orderResultsByScoreThenName); + nonMainIndexResults.sort(_orderResultsByScoreThenName); + + // Combine the result groups in (reverse) order. + // Non-main index entries are typically arbitrary cross-references, + // so display them after other results. + let results = [...nonMainIndexResults, ...normalResults]; // remove duplicate search results // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept @@ -361,7 +399,12 @@ const Search = { return acc; }, []); - results = results.reverse(); + return results.reverse(); + }, + + query: (query) => { + const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query); + const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms); // for debugging //Search.lastresults = results.slice(); // a copy @@ -466,14 +509,18 @@ const Search = { // add support for partial matches if (word.length > 2) { const escapedWord = _escapeRegExp(word); - Object.keys(terms).forEach((term) => { - if (term.match(escapedWord) && !terms[word]) - arr.push({ files: terms[term], score: Scorer.partialTerm }); - }); - Object.keys(titleTerms).forEach((term) => { - if (term.match(escapedWord) && !titleTerms[word]) - arr.push({ files: titleTerms[word], score: Scorer.partialTitle }); - }); + if (!terms.hasOwnProperty(word)) { + Object.keys(terms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: terms[term], score: Scorer.partialTerm }); + }); + } + if (!titleTerms.hasOwnProperty(word)) { + Object.keys(titleTerms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: titleTerms[term], score: Scorer.partialTitle }); + }); + } } // no match but word was a required one @@ -496,9 +543,8 @@ const Search = { // create the mapping files.forEach((file) => { - if (fileMap.has(file) && fileMap.get(file).indexOf(word) === -1) - fileMap.get(file).push(word); - else fileMap.set(file, [word]); + if (!fileMap.has(file)) fileMap.set(file, [word]); + else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word); }); }); @@ -549,8 +595,8 @@ const Search = { * search summary for a given text. keywords is a list * of stemmed words. */ - makeSearchSummary: (htmlText, keywords) => { - const text = Search.htmlToText(htmlText); + makeSearchSummary: (htmlText, keywords, anchor) => { + const text = Search.htmlToText(htmlText, anchor); if (text === "") return null; const textLower = text.toLowerCase(); diff --git a/docs/about.html b/docs/about.html index 210b740..c57aeaf 100644 --- a/docs/about.html +++ b/docs/about.html @@ -16,7 +16,7 @@ - + @@ -47,6 +47,7 @@