Merge pull request #1252 from msarahan/add_numpy_setup_py_data

add numpy setup.py data reader.  Fix bugs that this exposed.

.travis.yml

@@ -28,7 +28,7 @@ install:
   - conda update -q --all
   - conda install -q --force --no-deps conda requests
   - conda install -q pip pytest requests jinja2 patchelf flake8 mock python=$TRAVIS_PYTHON_VERSION pyflakes=1.1
-  - conda install -q anaconda-client pip pytest-cov
+  - conda install -q anaconda-client pip pytest-cov numpy
   - conda install -c conda-forge -q perl
   - pip install pytest-cov pytest-xdist pytest-capturelog filelock
   - if [[ "$CANARY" == "true" ]]; then

appveyor.yml

@@ -60,7 +60,7 @@ install:
   - python -c "import sys; print(sys.executable)"
   - python -c "import sys; print(sys.prefix)"
   - conda install -q pip pytest pytest-cov jinja2 patch flake8 mock requests
-  - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client
+  - conda install -q pyflakes=1.1 pycrypto posix m2-git anaconda-client numpy
   - conda install -c conda-forge -q perl
   # this is to ensure dependencies
   - python --version

conda_build/cli/main_build.py

@@ -25,6 +25,8 @@
 
 on_win = (sys.platform == 'win32')
 
+logging.basicConfig(level=logging.INFO)
+
 
 def parse_args(args):
     p = get_render_parser()
@@ -205,6 +207,7 @@ def execute(args):
     action = None
     if args.output:
         action = output_action
+        logging.basicConfig(level=logging.ERROR)
         config.verbose = False
         config.quiet = True
     elif args.test:
@@ -215,10 +218,6 @@ def execute(args):
         action = check_action
 
     if action:
-        if action == output_action:
-            logging.basicConfig(level=logging.ERROR)
-        else:
-            logging.basicConfig(level=logging.INFO)
         for recipe in args.recipe:
             recipe_dir, need_cleanup = get_recipe_abspath(recipe)
 

conda_build/environ.py

@@ -57,21 +57,29 @@ def get_sp_dir(config):
     return join(get_stdlib_dir(config), 'site-packages')
 
 
-def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
+def verify_git_repo(git_dir, git_url, config, expected_rev='HEAD'):
     env = os.environ.copy()
+    if config.verbose:
+        stderr = None
+    else:
+        FNULL = open(os.devnull, 'w')
+        stderr = FNULL
+        log.setLevel(logging.ERROR)
 
     if not expected_rev:
         return False
 
+    OK = True
+
     env['GIT_DIR'] = git_dir
     try:
         # Verify current commit matches expected commit
         current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"],
-                                      env=env, stderr=subprocess.STDOUT)
+                                      env=env, stderr=stderr)
         current_commit = current_commit.decode('utf-8')
         expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H",
                                             expected_rev],
-                                           env=env, stderr=subprocess.STDOUT)
+                                            env=env, stderr=stderr)
         expected_tag_commit = expected_tag_commit.decode('utf-8')
 
         if current_commit != expected_tag_commit:
@@ -80,7 +88,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
         # Verify correct remote url. Need to find the git cache directory,
         # and check the remote from there.
         cache_details = subprocess.check_output(["git", "remote", "-v"], env=env,
-                                     stderr=subprocess.STDOUT)
+                                     stderr=stderr)
         cache_details = cache_details.decode('utf-8')
         cache_dir = cache_details.split('\n')[0].split()[1]
 
@@ -91,13 +99,13 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
         try:
             remote_details = subprocess.check_output(["git", "--git-dir", cache_dir,
                                                       "remote", "-v"],
-                                                     env=env, stderr=subprocess.STDOUT)
+                                                     env=env, stderr=stderr)
         except subprocess.CalledProcessError:
             if sys.platform == 'win32' and cache_dir.startswith('/'):
                 cache_dir = utils.convert_unix_path_to_win(cache_dir)
             remote_details = subprocess.check_output(["git", "--git-dir", cache_dir,
                                                       "remote", "-v"],
-                                                     env=env, stderr=subprocess.STDOUT)
+                                                     env=env, stderr=stderr)
         remote_details = remote_details.decode('utf-8')
         remote_url = remote_details.split('\n')[0].split()[1]
 
@@ -113,18 +121,21 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
         # If the current source directory in conda-bld/work doesn't match the user's
         # metadata git_url or git_rev, then we aren't looking at the right source.
         if not os.path.isdir(remote_url) and remote_url.lower() != git_url.lower():
-            logging.debug("remote does not match git_url")
-            logging.debug("Remote: " + remote_url.lower())
-            logging.debug("git_url: " + git_url.lower())
-            return False
+            log.debug("remote does not match git_url")
+            log.debug("Remote: " + remote_url.lower())
+            log.debug("git_url: " + git_url.lower())
+            OK = False
     except subprocess.CalledProcessError as error:
-        logging.warn("Error obtaining git information.  Error was: ")
-        logging.warn(error)
-        return False
-    return True
+        log.warn("Error obtaining git information in verify_git_repo.  Error was: ")
+        log.warn(str(error))
+        OK = False
+    finally:
+        if not config.verbose:
+            FNULL.close()
+    return OK
 
 
-def get_git_info(repo):
+def get_git_info(repo, config):
     """
     Given a repo to a git repo, return a dictionary of:
       GIT_DESCRIBE_TAG
@@ -137,30 +148,40 @@ def get_git_info(repo):
     """
     d = {}
 
+    if config.verbose:
+        stderr = None
+    else:
+        FNULL = open(os.devnull, 'w')
+        stderr = FNULL
+        log.setLevel(logging.ERROR)
+
     # grab information from describe
     env = os.environ.copy()
     env['GIT_DIR'] = repo
     keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"]
 
-    output = subprocess.check_output(["git", "describe", "--tags", "--long", "HEAD"],
-                               env=env, cwd=os.path.dirname(repo))
-    output = output.decode('utf-8')
-
-    parts = output.rsplit('-', 2)
-    if len(parts) == 3:
-        d.update(dict(zip(keys, parts)))
-
-    # get the _full_ hash of the current HEAD
-    output = subprocess.check_output(["git", "rev-parse", "HEAD"],
-                                     env=env, cwd=os.path.dirname(repo))
-    output = output.decode('utf-8')
-
-    d['GIT_FULL_HASH'] = output
-    # set up the build string
-    if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d:
-        d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"],
-                                            d["GIT_DESCRIBE_HASH"])
-
+    try:
+        output = subprocess.check_output(["git", "describe", "--tags", "--long", "HEAD"],
+                                         env=env, cwd=os.path.dirname(repo), stderr=stderr)
+        output = output.decode('utf-8')
+
+        parts = output.rsplit('-', 2)
+        if len(parts) == 3:
+            d.update(dict(zip(keys, parts)))
+
+        # get the _full_ hash of the current HEAD
+        output = subprocess.check_output(["git", "rev-parse", "HEAD"],
+                                         env=env, cwd=os.path.dirname(repo), stderr=stderr)
+        output = output.decode('utf-8')
+
+        d['GIT_FULL_HASH'] = output
+        # set up the build string
+        if "GIT_DESCRIBE_NUMBER" in d and "GIT_DESCRIBE_HASH" in d:
+            d['GIT_BUILD_STR'] = '{}_{}'.format(d["GIT_DESCRIBE_NUMBER"],
+                                                d["GIT_DESCRIBE_HASH"])
+    except subprocess.CalledProcessError as error:
+        log.warn("Error obtaining git information in get_git_info.  Error was: ")
+        log.warn(str(error))
     return d
 
 
@@ -299,10 +320,11 @@ def meta_vars(meta, config):
         if git_url:
             _x = verify_git_repo(git_dir,
                                  git_url,
+                                 config,
                                  meta.get_value('source/git_rev', 'HEAD'))
 
         if _x or meta.get_value('source/path'):
-            d.update(get_git_info(git_dir))
+            d.update(get_git_info(git_dir, config))
 
     elif external.find_executable('hg', config.build_prefix) and os.path.exists(hg_dir):
         d.update(get_hg_build_info(hg_dir))

conda_build/jinja_context.py

@@ -111,6 +111,14 @@ def setup(**kw):
     # Patch setuptools, distutils
     setuptools_setup = setuptools.setup
     distutils_setup = distutils.core.setup
+    numpy_setup = None
+    try:
+        import numpy.distutils.core
+        numpy_setup = numpy.distutils.core.setup
+        numpy.distutils.core.setup = setup
+    except ImportError:
+        log.debug("Failed to import numpy for setup patch.  Is numpy installed?")
+
     setuptools.setup = distutils.core.setup = setup
     ns = {
         '__name__': '__main__',
@@ -120,8 +128,10 @@ def setup(**kw):
     if os.path.isfile(setup_file):
         code = compile(open(setup_file).read(), setup_file, 'exec', dont_inherit=1)
         exec(code, ns, ns)
-        distutils.core.setup = distutils_setup
-        setuptools.setup = setuptools_setup
+    distutils.core.setup = distutils_setup
+    setuptools.setup = setuptools_setup
+    if numpy_setup:
+        numpy.distutils.core.setup = numpy_setup
     if cd_to_work:
         os.chdir(cwd)
     # remove our workdir from sys.path

conda_build/render.py

@@ -87,7 +87,7 @@ def parse_or_try_download(metadata, no_download_source, config,
                 need_source_download = False
             try:
                 metadata.parse_again(config=config, permit_undefined_jinja=False)
-            except exceptions.UnableToParseMissingSetuptoolsDependencies:
+            except (ImportError, exceptions.UnableToParseMissingSetuptoolsDependencies):
                 need_reparse_in_env = True
         except subprocess.CalledProcessError as error:
             print("Warning: failed to download source.  If building, will try "
@@ -104,10 +104,11 @@ def parse_or_try_download(metadata, no_download_source, config,
         # we have not downloaded source in the render phase.  Download it in
         #     the build phase
         need_source_download = not no_download_source
-    try:
-        metadata.parse_until_resolved(config=config)
-    except exceptions.UnableToParseMissingSetuptoolsDependencies:
-        need_reparse_in_env = True
+    if not need_reparse_in_env:
+        try:
+            metadata.parse_until_resolved(config=config)
+        except exceptions.UnableToParseMissingSetuptoolsDependencies:
+            need_reparse_in_env = True
     return metadata, need_source_download, need_reparse_in_env
 
 

conda_build/utils.py

@@ -343,7 +343,6 @@ def silence_loggers(show_warnings_and_errors=True):
         log_level = logging.WARN
     else:
         log_level = logging.CRITICAL + 1
-    logging.disable(log_level)
     logging.getLogger(os.path.dirname(__file__)).setLevel(log_level)
     # This squelches a ton of conda output that is not hugely relevant
     logging.getLogger("conda").setLevel(log_level)

tests/test-recipes/metadata/_numpy_setup_py_data/meta.yaml

@@ -0,0 +1,18 @@
+package:
+  name: load_setup_py_test
+  version: {{ load_setup_py_data().version }}
+
+source:
+  git_url: https://github.com/NCAR/load_setup_py_test
+
+build:
+  number: 1
+  detect_binary_files_with_prefix: true
+
+requirements:
+  build:
+    - numpy x.x
+    - python
+  run:
+    - numpy x.x
+    - python

tests/test_api_build.py

@@ -15,7 +15,7 @@
 import pytest
 
 from conda_build import api
-from conda_build.utils import copy_into
+from conda_build.utils import copy_into, on_win
 
 from .utils import (metadata_dir, fail_dir, is_valid_dir, testing_workdir, test_config)
 
@@ -360,6 +360,7 @@ def test_compileall_compiles_all_good_files(testing_workdir, test_config):
 
 
 def test_render_setup_py_old_funcname(testing_workdir, test_config, caplog):
+    logging.basicConfig(level=logging.INFO)
     api.build(os.path.join(metadata_dir, "_source_setuptools"), config=test_config)
     assert "Deprecation notice: the load_setuptools function has been renamed to " in caplog.text()
 
@@ -386,6 +387,7 @@ def test_condarc_channel_available(testing_workdir, test_config):
 
 
 def test_debug_build_option(testing_workdir, test_config, caplog, capfd):
+    logging.basicConfig(level=logging.INFO)
     info_message = "Starting new HTTPS connection"
     debug_message = "GET /pkgs/free/noarch/repodata.json.bz2 HTTP/1.1"
     api.build(os.path.join(metadata_dir, "jinja2"), config=test_config)
@@ -394,9 +396,17 @@ def test_debug_build_option(testing_workdir, test_config, caplog, capfd):
     # this comes from a debug message
     assert debug_message not in caplog.text()
 
-    test_config.debug = True
-    api.build(os.path.join(metadata_dir, "jinja2"), config=test_config)
+    api.build(os.path.join(metadata_dir, "jinja2"), config=test_config, debug=True)
     # this comes from an info message
     assert info_message in caplog.text()
     # this comes from a debug message
     assert debug_message in caplog.text()
+
+
+@pytest.mark.skipif(on_win, reason="fortran compilers on win are hard.")
+def test_numpy_setup_py_data(test_config):
+    recipe_path = os.path.join(metadata_dir, '_numpy_setup_py_data')
+    assert os.path.basename(api.get_output_file_path(recipe_path,
+                            config=test_config, numpy="1.11")) == \
+                            "load_setup_py_test-1.0a1-np111py{0}{1}_1.tar.bz2".format(
+                                sys.version_info.major, sys.version_info.minor)