Skip to content

Commit

Permalink
Add --noarch-build-subdir option
Browse files Browse the repository at this point in the history
It would be nice to know that we can still build noarch and noarch python on other
OSes (well, except Windows at present). This allows that.

Also, in combination with --output-dir it allows building locally as would happen on
c3i. A sample script to glue this together is:

```
#!/usr/bin/env bash

set -e
FS_STUB=$1

__conda_setup="$(/opt/conda/bin/conda 'shell.bash' 'hook' 2> /dev/null)"
if [ $? -eq 0 ]; then
    eval "$__conda_setup"
fi

if [[ $(uname) == Darwin ]]; then
  BUILD_SUBDIR=osx-64
elif [[ $(uname) =~ M.* ]]; then
  BUILD_SUBDIR=win-64
else
  BUILD_SUBDIR=linux-64
fi

mkdir /tmp/$$.c3i-locally
pushd /tmp/$$.c3i-locally
  conda activate
  cp -rf /opt/r/a/$FS_STUB-feedstock .
  cp -f /opt/r/a/conda_build_config.yaml .
  c3i one-off rdonnelly.$FS_STUB  \
              $FS_STUB-feedstock  \
              --config-root-dir=/opt/asrc/automated-build/c3i_configurations/github_public  \
              --dry-run  \
              --output-dir $PWD  \
              --noarch-build-subdir ${BUILD_SUBDIR}  \
              --no-skip-existing
  mkdir rsync-recipes
  cp -rf *-on-* rsync-recipes/
  mkdir stats
  cmds=$(cat plan.yml | yq -r '.jobs[0].plan[] | select(.task=="build").config.run.args[1]')
  echo $cmds
  conda deactivate
  eval ${cmds}
popd
```
  • Loading branch information
mingwandroid committed Oct 31, 2020
1 parent 78c0241 commit 075d509
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 7 deletions.
6 changes: 5 additions & 1 deletion conda_concourse_ci/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,10 @@ def parse_args(parse_this=None):
"to all jobs. For finer control, use extra/worker_tags in "
"meta.yaml with selectors.",
dest='worker_tags')
one_off_parser.add_argument('--noarch-build-subdir', '-s', action='store', default='linux-64',
help="set the 'subdir' (machine to build on) for noarch",
dest='buildsubdir')

one_off_parser.add_argument(
'-m', '--variant-config-files',
action="append",
Expand Down Expand Up @@ -164,7 +168,7 @@ def parse_args(parse_this=None):
help="""File describing batch job. Each lines defines a seperate
one-off job. List one or more folders on each line. Job specific
arguments can be specified after a ';' using param=value, multiple
arguments are seperated by a ','. For example:
arguments are separated by a ','. For example:
recipe-feedstock; channel=conda-forge,clobber_sections_file=clobber.yaml
""")
Expand Down
13 changes: 7 additions & 6 deletions conda_concourse_ci/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ def collect_tasks(
append_sections_file=None,
pass_throughs=None,
skip_existing=True,
build_config_vars={}
build_config_vars={},
build_subdir='linux-64'
):
""" Return a graph of build tasks """
task_graph = nx.DiGraph()
Expand Down Expand Up @@ -126,19 +127,17 @@ def collect_tasks(
)
# merge this graph with the main one
task_graph = nx.compose(task_graph, graph)
collapse_noarch_python_nodes(task_graph)
collapse_noarch_python_nodes(task_graph, build_subdir=build_subdir)
return task_graph


def collapse_noarch_python_nodes(graph):
def collapse_noarch_python_nodes(graph, build_subdir='linux-64'):
""" Collapse nodes for noarch python packages into a single node
Collapse nodes corresponding to any noarch python packages so that each package
in built on a single platform and test on the remaining platforms. Edges are
reassinged or removed as needed.
"""
# TODO make build_subdir configurable
build_subdir = 'linux-64'

# find all noarch python builds, group by package name
noarch_groups = defaultdict(list)
Expand Down Expand Up @@ -536,6 +535,7 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None,
raise ValueError(
"--destroy-pipeline requires that --push-branch "
"and stage-for-upload be specified as well.")
build_subdir = kw.get('buildsubdir', 'linux-64')
folders = folders
path = path.replace('"', '')
if not folders:
Expand Down Expand Up @@ -591,7 +591,8 @@ def compute_builds(path, base_name, folders, matrix_base_dir=None,
clobber_sections_file=clobber_sections_file,
pass_throughs=pass_throughs,
skip_existing=skip_existing,
build_config_vars=build_config_vars
build_config_vars=build_config_vars,
build_subdir=build_subdir
)

with open(os.path.join(matrix_base_dir, 'config.yml')) as src:
Expand Down

0 comments on commit 075d509

Please sign in to comment.