diff --git a/conda-build-all.recipe/meta.yaml b/conda-build-all.recipe/meta.yaml index 00efda6..6265e91 100644 --- a/conda-build-all.recipe/meta.yaml +++ b/conda-build-all.recipe/meta.yaml @@ -20,6 +20,7 @@ requirements: - setuptools - conda - conda-build + - anaconda-client test: imports: diff --git a/conda_build_all/artefact_destination.py b/conda_build_all/artefact_destination.py index f754573..bfeafa0 100644 --- a/conda_build_all/artefact_destination.py +++ b/conda_build_all/artefact_destination.py @@ -10,6 +10,7 @@ import logging import os +import shutil import subprocess from argparse import Namespace @@ -48,6 +49,20 @@ def make_available(self, meta, built_dist_path, just_built): pass +class DirectoryDestination(ArtefactDestination): + def __init__(self, directory): + self.directory = os.path.abspath(os.path.expanduser(directory)) + if not os.path.exists(self.directory): + os.makedirs(self.directory) + if not os.path.isdir(self.directory): + raise IOError("The destination provided is not a directory.") + + def make_available(self, meta, built_dist_path, just_built): + if just_built: + print(meta, built_dist_path, just_built) + shutil.copy(built_dist_path, self.directory) + + class AnacondaClientChannelDest(ArtefactDestination): def __init__(self, token, owner, channel): self.token = token diff --git a/conda_build_all/builder.py b/conda_build_all/builder.py index e834303..f3b2459 100644 --- a/conda_build_all/builder.py +++ b/conda_build_all/builder.py @@ -46,21 +46,33 @@ def distribution_exists(binstar_cli, owner, metadata): return exists -def fetch_metas(directory): +def list_metas(directory, max_depth=0): """ Get the build metadata of all recipes in a directory. - The recipes will be sorted by the order of their directory name. + The order of metas from this function is not guaranteed. + + Parameters + ---------- + directory + Where to start looking for metas using os.walk. + max_depth : int + How deep to recurse when looking for recipes. + A value ``<=0`` will recurse indefinitely. A value of 1 + will look in the given directory for a meta.yaml. + (default: 0) """ packages = [] - for package_name in sorted(os.listdir(directory)): - package_dir = os.path.join(directory, package_name) - meta_yaml = os.path.join(package_dir, 'meta.yaml') - - if os.path.isdir(package_dir) and os.path.exists(meta_yaml): - packages.append(MetaData(package_dir)) - + current_depth = max_depth + root = os.path.normpath(directory) + for new_root, dirs, files in os.walk(root): + depth = new_root[len(root):].count(os.path.sep) + 1 + if max_depth > 0 and depth >= max_depth: + del dirs[:] + + if 'meta.yaml' in files: + packages.append(MetaData(new_root)) return packages @@ -117,7 +129,7 @@ def fetch_all_metas(self): """ conda_recipes_directory = os.path.abspath(os.path.expanduser(self.conda_recipes_directory)) - recipe_metas = fetch_metas(conda_recipes_directory) + recipe_metas = list_metas(conda_recipes_directory) recipe_metas = sort_dependency_order(recipe_metas) return recipe_metas @@ -149,7 +161,7 @@ def find_existing_built_dists(self, recipe_metas): def build(self, meta): print('Building ', meta.dist()) with meta.vn_context(): - build.build(meta.meta) + return bldpkg_path(build.build(meta.meta)) def compute_build_distros(self, index, recipes): """ diff --git a/conda_build_all/cli.py b/conda_build_all/cli.py index a403ba4..1954dbd 100644 --- a/conda_build_all/cli.py +++ b/conda_build_all/cli.py @@ -20,16 +20,11 @@ def main(): help='Skip a build if the equivalent disribution is already available in the specified directory.') parser.add_argument('--no-inspect-conda-bld-directory', default=True, action='store_false', help='Skip a build if the equivalent disribution is already in the conda-bld directory.') - parser.add_argument('--build-artefact-destinations', nargs='*', default=[], - help=('The channel(s) to upload built distributions to. It is ' - 'rare to specify this without the --inspect-channel argument. ' - 'If a file:// channel, the build will be copied to the directory. ' - 'If a url:// channel, the build will be uploaded with the anaconda ' - 'client functionality.')) + parser.add_argument('--artefact-directory', + help='A directory for any newly built distributions to be placed.') parser.add_argument('--upload-channels', nargs='*', default=[], help='The channel(s) to upload built distributions to.') - parser.add_argument("--matrix-conditions", nargs='*', default=[], help="Extra conditions for computing the build matrix.") parser.add_argument("--matrix-max-n-major-versions", default=2, type=int, @@ -56,6 +51,8 @@ def main(): artefact_destinations = [] for channel in args.upload_channels: artefact_destinations.append(artefact_dest.AnacondaClientChannelDest.from_spec(channel)) + if args.artefact_directory: + artefact_destinations.append(artefact_dest.DirectoryDestination(args.artefact_directory)) artefact_dest.log.setLevel(logging.INFO) artefact_dest.log.addHandler(logging.StreamHandler()) diff --git a/conda_build_all/tests/integration/test_builder.py b/conda_build_all/tests/integration/test_builder.py index 39409fa..9b4ac4c 100644 --- a/conda_build_all/tests/integration/test_builder.py +++ b/conda_build_all/tests/integration/test_builder.py @@ -7,6 +7,7 @@ from conda_build.metadata import MetaData +from conda_build_all.resolved_distribution import ResolvedDistribution from conda_build_all.builder import Builder from conda_build_all.tests.unit.dummy_index import DummyIndex @@ -25,12 +26,28 @@ def tearDown(self): def write_meta(self, recipe_dir_name, spec): recipe_dir = os.path.join(self.recipes_root_dir, recipe_dir_name) - os.mkdir(recipe_dir) + if not os.path.exists(recipe_dir): + os.makedirs(recipe_dir) with open(os.path.join(recipe_dir, 'meta.yaml'), 'w') as fh: fh.write(textwrap.dedent(spec)) return MetaData(recipe_dir) +class Test_build(RecipeCreatingUnit): + def test(self): + pkg1 = self.write_meta('pkg1', """ + package: + name: pkg1 + version: 1.0 + """) + pkg1_resolved = ResolvedDistribution(pkg1, (())) + builder = Builder(None, None, None, None, None) + r = builder.build(pkg1_resolved) + self.assertTrue(os.path.exists(r)) + self.assertEqual(os.path.abspath(r), r) + self.assertEqual(os.path.basename(r), 'pkg1-1.0-0.tar.bz2') + + class Test__find_existing_built_dists(RecipeCreatingUnit): def make_channel(self, all_metas): for meta in all_metas: diff --git a/conda_build_all/tests/unit/test_artefact_destination.py b/conda_build_all/tests/unit/test_artefact_destination.py index c54a356..9fbf430 100644 --- a/conda_build_all/tests/unit/test_artefact_destination.py +++ b/conda_build_all/tests/unit/test_artefact_destination.py @@ -3,13 +3,16 @@ import logging import mock import os +import shutil import sys +import tempfile import unittest from conda_build_all.tests.unit.dummy_index import DummyIndex, DummyPackage from conda_build_all.artefact_destination import (ArtefactDestination, - AnacondaClientChannelDest) + AnacondaClientChannelDest, + DirectoryDestination) import conda_build_all.artefact_destination @@ -110,5 +113,25 @@ def test_from_spec_owner_and_channel(self): self.assertEqual(dest.channel, 'my_channel') +class Test_DirectoryDestination(unittest.TestCase): + def setUp(self): + self.tmp_dir = tempfile.mkdtemp(prefix='recipes') + + def tearDown(self): + shutil.rmtree(self.tmp_dir) + + def test_not_copying(self): + dd = DirectoryDestination(self.tmp_dir) + dd.make_available(mock.sentinel.dummy_meta, mock.sentinel.dummy_path, + just_built=False) + + def test_copying(self): + dd = DirectoryDestination(self.tmp_dir) + with mock.patch('shutil.copy') as copy: + dd.make_available(mock.sentinel.dummy_meta, mock.sentinel.dummy_path, + just_built=True) + copy.assert_called_once_with(mock.sentinel.dummy_path, self.tmp_dir) + + if __name__ == '__main__': unittest.main() diff --git a/conda_build_all/tests/unit/test_builder.py b/conda_build_all/tests/unit/test_builder.py new file mode 100644 index 0000000..0c67482 --- /dev/null +++ b/conda_build_all/tests/unit/test_builder.py @@ -0,0 +1,61 @@ +import os +import shutil +import tempfile +import unittest +import textwrap + +import conda_build.config +from conda_build.metadata import MetaData + +from conda_build_all.builder import list_metas +from conda_build_all.tests.integration.test_builder import RecipeCreatingUnit + + +class Test_list_metas(RecipeCreatingUnit): + def setUp(self): + super(Test_list_metas, self).setUp() + m1 = self.write_meta('m1', """ + package: + name: m1 + """) + m2 = self.write_meta('.', """ + package: + name: m2 + """) + m3 = self.write_meta('d1/d2/d3/meta3', """ + package: + name: m3 + """) + m4 = self.write_meta('da1/da2/da3/meta4', """ + package: + name: m4 + """) + + def test_depth_0(self): + metas = list_metas(self.recipes_root_dir, max_depth=0) + names = [meta.name() for meta in metas] + self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4']) + + def test_depth_m1(self): + metas = list_metas(self.recipes_root_dir, max_depth=-1) + names = [meta.name() for meta in metas] + self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4']) + + def test_depth_1(self): + metas = list_metas(self.recipes_root_dir, max_depth=1) + names = [meta.name() for meta in metas] + self.assertEqual(sorted(names), ['m2']) + + def test_depth_2(self): + metas = list_metas(self.recipes_root_dir, max_depth=2) + names = [meta.name() for meta in metas] + self.assertEqual(sorted(names), ['m1', 'm2']) + + def test_default_depth(self): + metas = list_metas(self.recipes_root_dir) + names = [meta.name() for meta in metas] + self.assertEqual(sorted(names), ['m1', 'm2', 'm3', 'm4']) + + +if __name__ == '__main__': + unittest.main()