Merge branch 'activate_envs' into 1.21.x

appveyor.yml

@@ -53,18 +53,19 @@ install:
   - set "CONDA_ROOT=C:\Miniconda%BASE_PYTHON_VERSION%%ARCH_LABEL%"
   - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%"
   - conda config --set always_yes yes
+  # test with master of conda
   - conda update -q conda
-  - git clone https://github.com/conda/conda
-  - cd conda
-  - git checkout 4.0.8
-  - python setup.py install
-  - cd ../
+  #- git clone https://github.com/conda/conda
+  #- cd conda
+  #- git checkout 4.1.0
+  #- python setup.py install
+  #- cd ../
   - conda info
   - conda update -q --all
   - python -c "import sys; print(sys.version)"
   - python -c "import sys; print(sys.executable)"
   - python -c "import sys; print(sys.prefix)"
-  - conda install -q pytest pytest-cov git anaconda-client
+  - conda install -q pytest pytest-cov requests pycrypto git anaconda-client
   # this is to ensure dependencies
   - conda install -q conda-build
   - python --version

conda.recipe/meta.yaml

@@ -17,7 +17,7 @@ requirements:
   run:
     - python
     - psutil
-    - conda
+    - conda  >= 4.1
     - jinja2
     - patchelf      [linux]
 

conda_build/build.py

@@ -22,14 +22,15 @@
 from conda.api import get_index
 from conda.compat import PY3
 from conda.fetch import fetch_index
-from conda.install import prefix_placeholder, linked, move_to_trash
+from conda.install import prefix_placeholder, linked, move_to_trash, symlink_conda
 from conda.lock import Locked
 from conda.utils import url_path
 from conda.resolve import Resolve, MatchSpec, NoPackagesFound
 
 from conda_build import __version__
 from conda_build import environ, source, tarcheck
 from conda_build.config import config
+from conda_build.environ import activate_env, deactivate_env
 from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path
 from conda_build.scripts import create_entry_points, prepend_bin_path
 from conda_build.post import (post_process, post_build,
@@ -365,6 +366,11 @@ def create_env(prefix, specs, clear_cache=True):
     # ensure prefix exists, even if empty, i.e. when specs are empty
     if not isdir(prefix):
         os.makedirs(prefix)
+    if on_win:
+        shell = "cmd.exe"
+    else:
+        shell = "bash"
+    symlink_conda(prefix, sys.prefix, shell)
 
 
 def warn_on_old_conda_build(index):
@@ -462,22 +468,23 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
             create_env(config.build_prefix,
                     [ms.spec for ms in m.ms_depends('build')])
 
+            # modify os.environ to have our new environment be active
+            #    This is important for setting variables from activate.d and deactivate.d
+            # this modifies os.environ in place, and provides a diff of variables to actually
+            #    include in the build environment later
+            # Keeping all environment variables would defeat some isolation that we want.
+            env_diff = activate_env(config.build_prefix)
+
             if need_source_download:
                 # Execute any commands fetching the source (e.g., git) in the _build environment.
                 # This makes it possible to provide source fetchers (eg. git, hg, svn) as build
                 # dependencies.
-                _old_path = os.environ['PATH']
-                try:
-                    os.environ['PATH'] = prepend_bin_path({'PATH': _old_path},
-                                                            config.build_prefix)['PATH']
-                    m, need_source_download = parse_or_try_download(m,
-                                                                    no_download_source=False,
-                                                                    force_download=True,
-                                                                    verbose=verbose,
-                                                                    dirty=dirty)
-                    assert not need_source_download, "Source download failed.  Please investigate."
-                finally:
-                    os.environ['PATH'] = _old_path
+                m, need_source_download = parse_or_try_download(m,
+                                                                no_download_source=False,
+                                                                force_download=True,
+                                                                verbose=verbose,
+                                                                dirty=dirty)
+                assert not need_source_download, "Source download failed.  Please investigate."
 
             if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]:
                 print("%s is installed as a build dependency. Removing." %
@@ -528,9 +535,11 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
                     with open(join(source.get_dir(), 'bld.bat'), 'w') as bf:
                         bf.write(script)
                 import conda_build.windows as windows
-                windows.build(m, build_file, dirty=dirty)
+                windows.build(m, build_file, dirty=dirty, env_diff=env_diff)
             else:
                 env = environ.get_dict(m, dirty=dirty)
+                env.update(env_diff)
+
                 build_file = join(m.path, 'build.sh')
 
                 if script:
@@ -544,6 +553,10 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
 
                     _check_call(cmd, env=env, cwd=src_dir)
 
+        # this is necessary to return our current os.environ to normal.
+        #    We are discarding the list of changed values.
+        deactivate_env()
+
         if post in [True, None]:
             if post:
                 with open(join(config.croot, 'prefix_files.txt'), 'r') as f:
@@ -562,8 +575,8 @@ def build(m, post=None, include_recipe=True, keep_old_work=False,
             files2 = prefix_files()
             if any(config.meta_dir in join(config.build_prefix, f) for f in
                     files2 - files1):
-                sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory.  This error
-    usually comes from using conda in the build script.  Avoid doing this, as it
+                sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory.
+    This error usually comes from using conda in the build script.  Avoid doing this, as it
     can lead to packages that include their dependencies.""" %
                     (tuple(f for f in files2 - files1 if config.meta_dir in
                         join(config.build_prefix, f)),)))
@@ -677,9 +690,11 @@ def test(m, move_broken=True):
             specs += ['lua %s*' % environ.get_lua_ver()]
 
         create_env(config.test_prefix, specs)
+        env_diff = activate_env(config.test_prefix)
 
         env = dict(os.environ)
         env.update(environ.get_dict(m, prefix=config.test_prefix))
+        env.update(env_diff)
 
         # prepend bin (or Scripts) directory
         env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True)
@@ -733,6 +748,10 @@ def test(m, move_broken=True):
                 except subprocess.CalledProcessError:
                     tests_failed(m, move_broken=move_broken)
 
+        # this is necessary to return our current os.environ to normal.
+        #    We are discarding the list of changed values.
+        deactivate_env()
+
     print("TEST END:", m.dist())
 
 

conda_build/environ.py

@@ -7,10 +7,10 @@
 import warnings
 from collections import defaultdict
 from os.path import join, normpath
-from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE
+import subprocess
 
 import conda.config as cc
-from conda.compat import text_type
+from conda.compat import text_type, PY3
 
 from conda_build import external
 from conda_build import source
@@ -63,37 +63,41 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
     env['GIT_DIR'] = git_dir
     try:
         # Verify current commit matches expected commit
-        current_commit = check_output(["git", "log", "-n1", "--format=%H"],
-                                      env=env, stderr=STDOUT)
+        current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"],
+                                      env=env, stderr=subprocess.STDOUT)
         current_commit = current_commit.decode('utf-8')
-        expected_tag_commit = check_output(["git", "log", "-n1", "--format=%H",
+        expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H",
                                             expected_rev],
-                                           env=env, stderr=STDOUT)
+                                           env=env, stderr=subprocess.STDOUT)
         expected_tag_commit = expected_tag_commit.decode('utf-8')
 
         if current_commit != expected_tag_commit:
             return False
 
         # Verify correct remote url. Need to find the git cache directory,
         # and check the remote from there.
-        cache_details = check_output(["git", "remote", "-v"], env=env,
-                                     stderr=STDOUT)
+        cache_details = subprocess.check_output(["git", "remote", "-v"], env=env,
+                                     stderr=subprocess.STDOUT)
         cache_details = cache_details.decode('utf-8')
         cache_dir = cache_details.split('\n')[0].split()[1]
 
         if not isinstance(cache_dir, str):
             # On Windows, subprocess env can't handle unicode.
             cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8')
 
-        remote_details = check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env,
-                                                 stderr=STDOUT)
+        remote_details = subprocess.check_output(["git", "--git-dir", cache_dir, "remote", "-v"],
+                                                 env=env, stderr=subprocess.STDOUT)
         remote_details = remote_details.decode('utf-8')
         remote_url = remote_details.split('\n')[0].split()[1]
 
         # on windows, remote URL comes back to us as cygwin or msys format.  Python doesn't
         # know how to normalize it.  Need to convert it to a windows path.
         if sys.platform == 'win32' and remote_url.startswith('/'):
-            remote_url = check_output(["cygpath", '-w', remote_url]).rstrip().rstrip("\\")
+            cmd = "cygpath -w {0}".format(remote_url)
+            if PY3:
+                remote_url = subprocess.getoutput(cmd)
+            else:
+                remote_url = subprocess.check_output(cmd.split()).rstrip().rstrip("\\")
 
         if os.path.exists(remote_url):
             # Local filepaths are allowed, but make sure we normalize them
@@ -106,7 +110,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'):
             logging.debug("Remote: " + remote_url.lower() + "\n")
             logging.debug("git_url: " + git_url.lower() + "\n")
             return False
-    except CalledProcessError as error:
+    except subprocess.CalledProcessError as error:
         logging.warn("Error obtaining git information.  Error was: ")
         logging.warn(error)
         return False
@@ -131,8 +135,8 @@ def get_git_info(repo):
     env['GIT_DIR'] = repo
     keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"]
 
-    process = Popen(["git", "describe", "--tags", "--long", "HEAD"],
-                    stdout=PIPE, stderr=PIPE,
+    process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                     env=env)
     output = process.communicate()[0].strip()
     output = output.decode('utf-8')
@@ -142,8 +146,8 @@ def get_git_info(repo):
         d.update(dict(zip(keys, parts)))
 
     # get the _full_ hash of the current HEAD
-    process = Popen(["git", "rev-parse", "HEAD"],
-                    stdout=PIPE, stderr=PIPE, env=env)
+    process = subprocess.Popen(["git", "rev-parse", "HEAD"],
+                    stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env)
     output = process.communicate()[0].strip()
     output = output.decode('utf-8')
 
@@ -256,6 +260,12 @@ def meta_vars(meta):
         if os.path.exists(git_url):
             # If git_url is a relative path instead of a url, convert it to an abspath
             git_url = normpath(join(meta.path, git_url))
+            if sys.platform == 'win32':
+                cmd = "cygpath -w {0}".format(git_url)
+                if PY3:
+                    git_url = subprocess.getoutput(cmd)
+                else:
+                    git_url = subprocess.check_output(cmd.split()).rstrip().rstrip("\\")
 
         _x = False
 
@@ -279,8 +289,8 @@ def get_cpu_count():
     if sys.platform == "darwin":
         # multiprocessing.cpu_count() is not reliable on OSX
         # See issue #645 on github.com/conda/conda-build
-        out, err = Popen('sysctl -n hw.logicalcpu', shell=True,
-                         stdout=PIPE).communicate()
+        out, err = subprocess.Popen('sysctl -n hw.logicalcpu', shell=True,
+                         stdout=subprocess.PIPE).communicate()
         return out.decode('utf-8').strip()
     else:
         try:
@@ -366,6 +376,70 @@ def system_vars(env_dict, prefix):
     return d
 
 
+# http://code.activestate.com/recipes/576644-diff-two-dictionaries/#c9
+def _dict_diff(d1, d2):
+    """Shows entries that have changed or been added in d2 relative to d1"""
+    both = set(d1.keys()) & set(d2.keys())
+    diff = {k: d2[k] for k in both if d1[k] != d2[k]}
+    diff.update({k: d2[k] for k in set(d2.keys()) - both})
+    return diff
+
+
+def _set_environ_from_subprocess_values(vars):
+    """
+    Vars is an unprocessed string of envrionment variable output, such as from ```set```
+    on Windows, or ```env``` elsewhere
+    """
+    start_environ = os.environ
+    vars = vars.split("\n")
+    modified_environ = {var.split("=")[0].strip(): var.split("=")[1].strip()
+                        for var in vars if "=" in var}
+
+    # the diff is the only part we'll set for the actual build environment - mind you,
+    #   the current process is not the actual build environment.  That is always a native
+    #   shell subprocess.
+    diff = _dict_diff(start_environ, modified_environ)
+
+    # modify the current process with the activated/deactivated values
+    for key, value in modified_environ.items():
+        os.environ[key] = value
+
+    return diff
+
+
+def activate_or_deactivate_env(action, env_name_or_path=""):
+    """
+    Strategy is to open a subprocess, run activate, record the variables,
+    then apply them in our process
+
+    action should be "activate" or "deactivate"
+    env_name_or_path should only be provided for activation.
+    """
+    if sys.platform == "win32":
+        cmd = '"{0}\\Scripts\\{1}.bat" "{2}" && set'.format(sys.prefix,
+                                                            action,
+                                                            env_name_or_path)
+        if PY3:
+            # this method simplifies dealing with str vs bytestring on Py3.
+            vars = subprocess.getoutput(cmd).replace("\r\n", "\n")
+        else:
+            vars = subprocess.check_output(cmd).replace("\r\n", "\n")
+    else:
+        cmd = "source {0}/bin/{1} {2} && env".format(sys.prefix, action, env_name_or_path)
+        vars = subprocess.check_output(["bash", "-c", cmd], env=os.environ)
+    if PY3 and hasattr(vars, "decode"):
+        vars = vars.decode("UTF-8")
+    return _set_environ_from_subprocess_values(vars)
+
+
+def activate_env(env_name_or_path):
+    return activate_or_deactivate_env("activate", env_name_or_path)
+
+
+def deactivate_env():
+    return activate_or_deactivate_env("deactivate")
+
+
 if __name__ == '__main__':
     e = get_dict()
     for k in sorted(e):

conda_build/pypi.py

@@ -759,8 +759,9 @@ def run_setuppy(src_dir, temp_dir, python_version):
     # haywire.
     # TODO: Try with another version of Python if this one fails. Some
     # packages are Python 2 or Python 3 only.
-    create_env(config.build_prefix, ['python %s*' % python_version, 'pyyaml',
-        'setuptools', 'numpy'], clear_cache=False)
+    create_env(config.build_prefix, ['python %s*' % python_version,
+                                     'pyyaml', 'yaml',
+                                     'setuptools', 'numpy'], clear_cache=False)
     stdlib_dir = join(config.build_prefix,
                       'Lib' if sys.platform == 'win32'
                       else 'lib/python%s' % python_version)

conda_build/windows.py

@@ -186,10 +186,12 @@ def kill_processes(process_names=["msbuild.exe"]):
             continue
 
 
-def build(m, bld_bat, dirty=False):
-    env = dict(os.environ)
-    env.update(environ.get_dict(m, dirty=dirty))
-    env = environ.prepend_bin_path(env, config.build_prefix, True)
+def build(m, bld_bat, dirty=False, env_diff=None):
+    if not env_diff:
+        env_diff = {}
+    env = environ.get_dict(m, dirty=dirty)
+    # env_diff contains activation info, among any other env variables set by activation scripts
+    env.update(env_diff)
 
     for name in 'BIN', 'INC', 'LIB':
         path = env['LIBRARY_' + name]

setup.py

@@ -34,8 +34,8 @@
         "Programming Language :: Python :: 2",
         "Programming Language :: Python :: 2.7",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
     ],
     description="tools for building conda packages",
     long_description=open('README.rst').read(),

tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat

@@ -0,0 +1,5 @@
+mkdir %PREFIX%\etc\conda\activate.d
+echo set TEST_VAR=1 > %PREFIX%\etc\conda\activate.d\test.bat
+
+mkdir %PREFIX%\etc\conda\deactivate.d
+echo set TEST_VAR= > %PREFIX%\etc\conda\deactivate.d\test.bat