From 075d509a8f62e92c2898e6923693199716bff6b3 Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Sat, 31 Oct 2020 11:54:50 +0100 Subject: [PATCH] Add --noarch-build-subdir option It would be nice to know that we can still build noarch and noarch python on other OSes (well, except Windows at present). This allows that. Also, in combination with --output-dir it allows building locally as would happen on c3i. A sample script to glue this together is: ``` #!/usr/bin/env bash set -e FS_STUB=$1 __conda_setup="$(/opt/conda/bin/conda 'shell.bash' 'hook' 2> /dev/null)" if [ $? -eq 0 ]; then eval "$__conda_setup" fi if [[ $(uname) == Darwin ]]; then BUILD_SUBDIR=osx-64 elif [[ $(uname) =~ M.* ]]; then BUILD_SUBDIR=win-64 else BUILD_SUBDIR=linux-64 fi mkdir /tmp/$$.c3i-locally pushd /tmp/$$.c3i-locally conda activate cp -rf /opt/r/a/$FS_STUB-feedstock . cp -f /opt/r/a/conda_build_config.yaml . c3i one-off rdonnelly.$FS_STUB \ $FS_STUB-feedstock \ --config-root-dir=/opt/asrc/automated-build/c3i_configurations/github_public \ --dry-run \ --output-dir $PWD \ --noarch-build-subdir ${BUILD_SUBDIR} \ --no-skip-existing mkdir rsync-recipes cp -rf *-on-* rsync-recipes/ mkdir stats cmds=$(cat plan.yml | yq -r '.jobs[0].plan[] | select(.task=="build").config.run.args[1]') echo $cmds conda deactivate eval ${cmds} popd ``` --- conda_concourse_ci/cli.py | 6 +++++- conda_concourse_ci/execute.py | 13 +++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/conda_concourse_ci/cli.py b/conda_concourse_ci/cli.py index b5386c95..f2c53a48 100644 --- a/conda_concourse_ci/cli.py +++ b/conda_concourse_ci/cli.py @@ -114,6 +114,10 @@ def parse_args(parse_this=None): "to all jobs. For finer control, use extra/worker_tags in " "meta.yaml with selectors.", dest='worker_tags') + one_off_parser.add_argument('--noarch-build-subdir', '-s', action='store', default='linux-64', + help="set the 'subdir' (machine to build on) for noarch", + dest='buildsubdir') + one_off_parser.add_argument( '-m', '--variant-config-files', action="append", @@ -164,7 +168,7 @@ def parse_args(parse_this=None): help="""File describing batch job. Each lines defines a seperate one-off job. List one or more folders on each line. Job specific arguments can be specified after a ';' using param=value, multiple - arguments are seperated by a ','. For example: + arguments are separated by a ','. For example: recipe-feedstock; channel=conda-forge,clobber_sections_file=clobber.yaml """) diff --git a/conda_concourse_ci/execute.py b/conda_concourse_ci/execute.py index 31aba436..49b3d854 100644 --- a/conda_concourse_ci/execute.py +++ b/conda_concourse_ci/execute.py @@ -80,7 +80,8 @@ def collect_tasks( append_sections_file=None, pass_throughs=None, skip_existing=True, - build_config_vars={} + build_config_vars={}, + build_subdir='linux-64' ): """ Return a graph of build tasks """ task_graph = nx.DiGraph() @@ -126,19 +127,17 @@ def collect_tasks( ) # merge this graph with the main one task_graph = nx.compose(task_graph, graph) - collapse_noarch_python_nodes(task_graph) + collapse_noarch_python_nodes(task_graph, build_subdir=build_subdir) return task_graph -def collapse_noarch_python_nodes(graph): +def collapse_noarch_python_nodes(graph, build_subdir='linux-64'): """ Collapse nodes for noarch python packages into a single node Collapse nodes corresponding to any noarch python packages so that each package in built on a single platform and test on the remaining platforms. Edges are reassinged or removed as needed. """ - # TODO make build_subdir configurable - build_subdir = 'linux-64' # find all noarch python builds, group by package name noarch_groups = defaultdict(list) @@ -536,6 +535,7 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None, raise ValueError( "--destroy-pipeline requires that --push-branch " "and stage-for-upload be specified as well.") + build_subdir = kw.get('buildsubdir', 'linux-64') folders = folders path = path.replace('"', '') if not folders: @@ -591,7 +591,8 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None, clobber_sections_file=clobber_sections_file, pass_throughs=pass_throughs, skip_existing=skip_existing, - build_config_vars=build_config_vars + build_config_vars=build_config_vars, + build_subdir=build_subdir ) with open(os.path.join(matrix_base_dir, 'config.yml')) as src: