From d667703f115f676b0ce23e5a920f18c9a6a33e94 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 Nov 2017 21:47:15 -0500 Subject: [PATCH 1/3] adaptations for conda-build 3 --- .travis.yml | 4 +- conda_build_all/builder.py | 20 ++++++++-- conda_build_all/conda_interface.py | 3 ++ .../tests/integration/test_builder.py | 38 ++++++++++++++----- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6c45ba7..2be6a97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ env: - PYTHON=3.5 CONDA_BUILD_ALL_TEST_ANACONDA_CLOUD=1 - PYTHON=3.5 - CONDA_ORIGIN=defaults + CONDA_ORIGIN=https://repo.continuum.io/pkgs/main install: - mkdir -p ${HOME}/cache/pkgs @@ -37,7 +37,7 @@ install: # Now do the things we need to do to install it. - conda install -c conda-forge --file requirements.txt nose mock python=${PYTHON} ${EXTRA_DEPS} --yes --quiet - - if [[ -n ${CONDA_ORIGIN} ]]; then conda install -yq -f -c ${CONDA_ORIGIN} conda; fi + - if [[ -n ${CONDA_ORIGIN} ]]; then conda install -yq -c ${CONDA_ORIGIN} conda conda-build; fi - python setup.py install - mkdir not_the_source_root && cd not_the_source_root diff --git a/conda_build_all/builder.py b/conda_build_all/builder.py index 197de12..4f6fd8a 100644 --- a/conda_build_all/builder.py +++ b/conda_build_all/builder.py @@ -84,7 +84,13 @@ def list_metas(directory, max_depth=0, config=None): if 'meta.yaml' in files: if hasattr(conda_build, 'api'): - packages.append(conda_build.api.render(new_root, config=config)[0]) + pkgs = conda_build.api.render(new_root, config=config, + finalize=False, bypass_env_check=True) + if hasattr(pkgs[0], 'config'): + pkgs = [pkgs[0]] + else: + pkgs = [pkg[0] for pkg in pkgs] + packages.extend(pkgs) else: packages.append(MetaData(new_root)) @@ -107,8 +113,15 @@ def select_lines(data, *args, **kwargs): # is only a suitable solution when selectors are also comments. return data + meta.final = False with mock.patch('conda_build.metadata.select_lines', new=select_lines): - with mock.patch('conda_build.jinja_context.select_lines', new=select_lines): + try: + with mock.patch('conda_build.jinja_context.select_lines', new=select_lines): + try: + meta.parse_again(config, permit_undefined_jinja=True) + except TypeError: + meta.parse_again(permit_undefined_jinja=True) + except AttributeError: try: meta.parse_again(config, permit_undefined_jinja=True) except TypeError: @@ -206,8 +219,7 @@ def build(self, meta, config): print('Building ', meta.dist()) config = meta.vn_context(config=config) try: - conda_build.api.build(meta.meta, config=config) - return conda_build.api.get_output_file_path(meta.meta, config) + return conda_build.api.build(meta.meta, config=config) except AttributeError: with meta.vn_context(): return bldpkg_path(build.build(meta.meta)) diff --git a/conda_build_all/conda_interface.py b/conda_build_all/conda_interface.py index c56c24d..a8bd663 100644 --- a/conda_build_all/conda_interface.py +++ b/conda_build_all/conda_interface.py @@ -13,6 +13,7 @@ from conda.exports import Unsatisfiable from conda.exports import NoPackagesFound from conda.exports import Resolve + from conda.exports import string_types from conda.models.dist import Dist as _Dist def get_key(dist_or_filename): @@ -37,6 +38,7 @@ def ensure_dist_or_dict(fn): from conda.exports import Unsatisfiable from conda.exports import NoPackagesFound from conda.exports import Resolve + from conda.exports import string_types def get_key(dist_or_filename): return dist_or_filename.fn @@ -63,3 +65,4 @@ def ensure_dist_or_dict(fn): Resolve, get_index = Resolve, get_index MatchSpec = MatchSpec Unsatisfiable, NoPackagesFound = Unsatisfiable, NoPackagesFound +string_types = string_types diff --git a/conda_build_all/tests/integration/test_builder.py b/conda_build_all/tests/integration/test_builder.py index ca3b5f0..6538651 100644 --- a/conda_build_all/tests/integration/test_builder.py +++ b/conda_build_all/tests/integration/test_builder.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import os +import re import shutil import tempfile import textwrap @@ -11,6 +12,7 @@ import conda_build.config from conda_build.metadata import MetaData +from conda_build_all.conda_interface import string_types from conda_build_all.resolved_distribution import ResolvedDistribution from conda_build_all.builder import Builder @@ -58,9 +60,15 @@ def test_no_source(self): r = builder.build(pkg1_resolved, conda_build.api.Config()) else: r = builder.build(pkg1_resolved, conda_build.config.config) - self.assertTrue(os.path.exists(r)) - self.assertEqual(os.path.abspath(r), r) - self.assertEqual(os.path.basename(r), 'pkg1-1.0-0.tar.bz2') + if isinstance(r, string_types): + rs = [r] + else: + rs = r + for r in rs: + self.assertTrue(os.path.exists(r)) + self.assertEqual(os.path.abspath(r), r) + self.assertTrue(bool(re.match('pkg1-1.0-(h[0-9a-f]{7}_)?0.tar.bz2', + os.path.basename(r)))) def test_noarch_python(self): pkg1 = self.write_meta('pkg1', """ @@ -81,9 +89,15 @@ def test_noarch_python(self): r = builder.build(pkg1_resolved, conda_build.api.Config()) else: r = builder.build(pkg1_resolved, conda_build.config.config) - self.assertTrue(os.path.exists(r)) - self.assertEqual(os.path.abspath(r), r) - self.assertEqual(os.path.basename(r), 'pkg1-1.0-py_0.tar.bz2') + if isinstance(r, string_types): + rs = [r] + else: + rs = r + for r in rs: + self.assertTrue(os.path.exists(r)) + self.assertEqual(os.path.abspath(r), r) + self.assertTrue(bool(re.match('pkg1-1.0-py(h[0-9a-f]{7})?_0.tar.bz2', + os.path.basename(r)))) def test_numpy_dep(self): pkg1 = self.write_meta('pkg1', """ @@ -104,9 +118,15 @@ def test_numpy_dep(self): r = builder.build(pkg1_resolved, conda_build.api.Config()) else: r = builder.build(pkg1_resolved, conda_build.config.config) - self.assertTrue(os.path.exists(r)) - self.assertEqual(os.path.abspath(r), r) - self.assertEqual(os.path.basename(r), 'pkg1-1.0-np111py35_0.tar.bz2') + if isinstance(r, string_types): + rs = [r] + else: + rs = r + for r in rs: + self.assertTrue(os.path.exists(r)) + self.assertEqual(os.path.abspath(r), r) + self.assertTrue(bool(re.match('pkg1-1.0-np111py35(h[0-9a-f]{7})?_0.tar.bz2', + os.path.basename(r)))) class Test__find_existing_built_dists(RecipeCreatingUnit): From 247af126749bb7a17625e8e626ab7322f2488514 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 2 Nov 2017 18:11:05 -0500 Subject: [PATCH 2/3] change travis to test against cb2.1; simplify build output handling --- .travis.yml | 2 +- conda_build_all/builder.py | 10 +++++--- .../tests/integration/test_builder.py | 24 +++++-------------- 3 files changed, 14 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2be6a97..14604e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ env: - PYTHON=2.7 - PYTHON=3.4 - PYTHON=3.5 - EXTRA_DEPS='conda-build=1.*' + EXTRA_DEPS='conda-build=2.1.*' - PYTHON=3.5 CONDA_BUILD_ALL_TEST_ANACONDA_CLOUD=1 - PYTHON=3.5 diff --git a/conda_build_all/builder.py b/conda_build_all/builder.py index 4f6fd8a..a1222f8 100644 --- a/conda_build_all/builder.py +++ b/conda_build_all/builder.py @@ -19,7 +19,8 @@ from binstar_client.utils import get_binstar import binstar_client -from .conda_interface import Resolve, get_index, subdir, copy_index +from .conda_interface import (Resolve, get_index, subdir, copy_index, + string_types) try: import conda_build.api @@ -219,10 +220,13 @@ def build(self, meta, config): print('Building ', meta.dist()) config = meta.vn_context(config=config) try: - return conda_build.api.build(meta.meta, config=config) + output_paths = conda_build.api.build(meta.meta, config=config) except AttributeError: with meta.vn_context(): - return bldpkg_path(build.build(meta.meta)) + output_paths = bldpkg_path(build.build(meta.meta)) + if isinstance(output_paths, string_types): + output_paths = [output_paths] + return output_paths def compute_build_distros(self, index, recipes, config): """ diff --git a/conda_build_all/tests/integration/test_builder.py b/conda_build_all/tests/integration/test_builder.py index 6538651..0e84cf4 100644 --- a/conda_build_all/tests/integration/test_builder.py +++ b/conda_build_all/tests/integration/test_builder.py @@ -57,13 +57,9 @@ def test_no_source(self): pkg1_resolved = ResolvedDistribution(pkg1, (())) builder = Builder(None, None, None, None, None) if hasattr(conda_build, 'api'): - r = builder.build(pkg1_resolved, conda_build.api.Config()) + rs = builder.build(pkg1_resolved, conda_build.api.Config()) else: - r = builder.build(pkg1_resolved, conda_build.config.config) - if isinstance(r, string_types): - rs = [r] - else: - rs = r + rs = builder.build(pkg1_resolved, conda_build.config.config) for r in rs: self.assertTrue(os.path.exists(r)) self.assertEqual(os.path.abspath(r), r) @@ -86,13 +82,9 @@ def test_noarch_python(self): pkg1_resolved = ResolvedDistribution(pkg1, (['python', '3.5'], )) builder = Builder(None, None, None, None, None) if hasattr(conda_build, 'api'): - r = builder.build(pkg1_resolved, conda_build.api.Config()) - else: - r = builder.build(pkg1_resolved, conda_build.config.config) - if isinstance(r, string_types): - rs = [r] + rs = builder.build(pkg1_resolved, conda_build.api.Config()) else: - rs = r + rs = builder.build(pkg1_resolved, conda_build.config.config) for r in rs: self.assertTrue(os.path.exists(r)) self.assertEqual(os.path.abspath(r), r) @@ -115,13 +107,9 @@ def test_numpy_dep(self): pkg1_resolved = ResolvedDistribution(pkg1, (['python', '3.5'], ['numpy', '1.11'])) builder = Builder(None, None, None, None, None) if hasattr(conda_build, 'api'): - r = builder.build(pkg1_resolved, conda_build.api.Config()) - else: - r = builder.build(pkg1_resolved, conda_build.config.config) - if isinstance(r, string_types): - rs = [r] + rs = builder.build(pkg1_resolved, conda_build.api.Config()) else: - rs = r + rs = builder.build(pkg1_resolved, conda_build.config.config) for r in rs: self.assertTrue(os.path.exists(r)) self.assertEqual(os.path.abspath(r), r) From 9a47d252f37c259f2226e70ffc615c29c4f76f41 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 2 Nov 2017 18:17:35 -0500 Subject: [PATCH 3/3] change older conda-build ver to 2.0.*; comment on metadata obj collection --- .travis.yml | 3 ++- conda_build_all/builder.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 14604e2..2e8ee09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,8 +14,9 @@ env: matrix: - PYTHON=2.7 - PYTHON=3.4 + # test against older conda-build - PYTHON=3.5 - EXTRA_DEPS='conda-build=2.1.*' + EXTRA_DEPS='conda-build=2.0.*' - PYTHON=3.5 CONDA_BUILD_ALL_TEST_ANACONDA_CLOUD=1 - PYTHON=3.5 diff --git a/conda_build_all/builder.py b/conda_build_all/builder.py index a1222f8..5395324 100644 --- a/conda_build_all/builder.py +++ b/conda_build_all/builder.py @@ -87,8 +87,12 @@ def list_metas(directory, max_depth=0, config=None): if hasattr(conda_build, 'api'): pkgs = conda_build.api.render(new_root, config=config, finalize=False, bypass_env_check=True) + # cb2 returns a tuple, with the metadata object as the first + # element. That's all we care about. if hasattr(pkgs[0], 'config'): pkgs = [pkgs[0]] + # cb3 returns a list of tuples, each with the metadata object + # as the first element. Collect them up. else: pkgs = [pkg[0] for pkg in pkgs] packages.extend(pkgs)