diff --git a/appveyor.yml b/appveyor.yml index a5ef291591..cf086a1a4d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -53,18 +53,19 @@ install: - set "CONDA_ROOT=C:\Miniconda%BASE_PYTHON_VERSION%%ARCH_LABEL%" - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - conda config --set always_yes yes + # test with master of conda - conda update -q conda - - git clone https://github.com/conda/conda - - cd conda - - git checkout 4.0.8 - - python setup.py install - - cd ../ + #- git clone https://github.com/conda/conda + #- cd conda + #- git checkout 4.1.0 + #- python setup.py install + #- cd ../ - conda info - conda update -q --all - python -c "import sys; print(sys.version)" - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - - conda install -q pytest pytest-cov git anaconda-client + - conda install -q pytest pytest-cov requests pycrypto git anaconda-client # this is to ensure dependencies - conda install -q conda-build - python --version diff --git a/bin/conda-render b/bin/conda-render old mode 100644 new mode 100755 diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index 49ad5d5cb3..cf1e96a4f7 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -17,7 +17,7 @@ requirements: run: - python - psutil - - conda + - conda >= 4.1 - jinja2 - patchelf [linux] diff --git a/conda_build/build.py b/conda_build/build.py index e4ddae7f34..adb343858d 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -22,7 +22,7 @@ from conda.api import get_index from conda.compat import PY3 from conda.fetch import fetch_index -from conda.install import prefix_placeholder, linked, move_to_trash +from conda.install import prefix_placeholder, linked, move_to_trash, symlink_conda from conda.lock import Locked from conda.utils import url_path from conda.resolve import Resolve, MatchSpec, NoPackagesFound @@ -30,6 +30,7 @@ from conda_build import __version__ from conda_build import environ, source, tarcheck from conda_build.config import config +from conda_build.environ import activate_env, deactivate_env from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path from conda_build.scripts import create_entry_points, prepend_bin_path from conda_build.post import (post_process, post_build, @@ -365,6 +366,11 @@ def create_env(prefix, specs, clear_cache=True): # ensure prefix exists, even if empty, i.e. when specs are empty if not isdir(prefix): os.makedirs(prefix) + if on_win: + shell = "cmd.exe" + else: + shell = "bash" + symlink_conda(prefix, sys.prefix, shell) def warn_on_old_conda_build(index): @@ -462,22 +468,23 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, create_env(config.build_prefix, [ms.spec for ms in m.ms_depends('build')]) + # modify os.environ to have our new environment be active + # This is important for setting variables from activate.d and deactivate.d + # this modifies os.environ in place, and provides a diff of variables to actually + # include in the build environment later + # Keeping all environment variables would defeat some isolation that we want. + env_diff = activate_env(config.build_prefix) + if need_source_download: # Execute any commands fetching the source (e.g., git) in the _build environment. # This makes it possible to provide source fetchers (eg. git, hg, svn) as build # dependencies. - _old_path = os.environ['PATH'] - try: - os.environ['PATH'] = prepend_bin_path({'PATH': _old_path}, - config.build_prefix)['PATH'] - m, need_source_download = parse_or_try_download(m, - no_download_source=False, - force_download=True, - verbose=verbose, - dirty=dirty) - assert not need_source_download, "Source download failed. Please investigate." - finally: - os.environ['PATH'] = _old_path + m, need_source_download = parse_or_try_download(m, + no_download_source=False, + force_download=True, + verbose=verbose, + dirty=dirty) + assert not need_source_download, "Source download failed. Please investigate." if m.name() in [i.rsplit('-', 2)[0] for i in linked(config.build_prefix)]: print("%s is installed as a build dependency. Removing." % @@ -528,9 +535,11 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, with open(join(source.get_dir(), 'bld.bat'), 'w') as bf: bf.write(script) import conda_build.windows as windows - windows.build(m, build_file, dirty=dirty) + windows.build(m, build_file, dirty=dirty, env_diff=env_diff) else: env = environ.get_dict(m, dirty=dirty) + env.update(env_diff) + build_file = join(m.path, 'build.sh') if script: @@ -544,6 +553,10 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, _check_call(cmd, env=env, cwd=src_dir) + # this is necessary to return our current os.environ to normal. + # We are discarding the list of changed values. + deactivate_env() + if post in [True, None]: if post: with open(join(config.croot, 'prefix_files.txt'), 'r') as f: @@ -562,8 +575,8 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, files2 = prefix_files() if any(config.meta_dir in join(config.build_prefix, f) for f in files2 - files1): - sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory. This error - usually comes from using conda in the build script. Avoid doing this, as it + sys.exit(indent("""Error: Untracked file(s) %s found in conda-meta directory. + This error usually comes from using conda in the build script. Avoid doing this, as it can lead to packages that include their dependencies.""" % (tuple(f for f in files2 - files1 if config.meta_dir in join(config.build_prefix, f)),))) @@ -677,9 +690,11 @@ def test(m, move_broken=True): specs += ['lua %s*' % environ.get_lua_ver()] create_env(config.test_prefix, specs) + env_diff = activate_env(config.test_prefix) env = dict(os.environ) env.update(environ.get_dict(m, prefix=config.test_prefix)) + env.update(env_diff) # prepend bin (or Scripts) directory env = prepend_bin_path(env, config.test_prefix, prepend_prefix=True) @@ -733,6 +748,10 @@ def test(m, move_broken=True): except subprocess.CalledProcessError: tests_failed(m, move_broken=move_broken) + # this is necessary to return our current os.environ to normal. + # We are discarding the list of changed values. + deactivate_env() + print("TEST END:", m.dist()) diff --git a/conda_build/environ.py b/conda_build/environ.py index 55bbd3e41f..7ff876ee83 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -7,10 +7,10 @@ import warnings from collections import defaultdict from os.path import join, normpath -from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE +import subprocess import conda.config as cc -from conda.compat import text_type +from conda.compat import text_type, PY3 from conda_build import external from conda_build import source @@ -63,12 +63,12 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): env['GIT_DIR'] = git_dir try: # Verify current commit matches expected commit - current_commit = check_output(["git", "log", "-n1", "--format=%H"], - env=env, stderr=STDOUT) + current_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H"], + env=env, stderr=subprocess.STDOUT) current_commit = current_commit.decode('utf-8') - expected_tag_commit = check_output(["git", "log", "-n1", "--format=%H", + expected_tag_commit = subprocess.check_output(["git", "log", "-n1", "--format=%H", expected_rev], - env=env, stderr=STDOUT) + env=env, stderr=subprocess.STDOUT) expected_tag_commit = expected_tag_commit.decode('utf-8') if current_commit != expected_tag_commit: @@ -76,8 +76,8 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): # Verify correct remote url. Need to find the git cache directory, # and check the remote from there. - cache_details = check_output(["git", "remote", "-v"], env=env, - stderr=STDOUT) + cache_details = subprocess.check_output(["git", "remote", "-v"], env=env, + stderr=subprocess.STDOUT) cache_details = cache_details.decode('utf-8') cache_dir = cache_details.split('\n')[0].split()[1] @@ -85,15 +85,19 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): # On Windows, subprocess env can't handle unicode. cache_dir = cache_dir.encode(sys.getfilesystemencoding() or 'utf-8') - remote_details = check_output(["git", "--git-dir", cache_dir, "remote", "-v"], env=env, - stderr=STDOUT) + remote_details = subprocess.check_output(["git", "--git-dir", cache_dir, "remote", "-v"], + env=env, stderr=subprocess.STDOUT) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] # on windows, remote URL comes back to us as cygwin or msys format. Python doesn't # know how to normalize it. Need to convert it to a windows path. if sys.platform == 'win32' and remote_url.startswith('/'): - remote_url = check_output(["cygpath", '-w', remote_url]).rstrip().rstrip("\\") + cmd = "cygpath -w {0}".format(remote_url) + if PY3: + remote_url = subprocess.getoutput(cmd) + else: + remote_url = subprocess.check_output(cmd.split()).rstrip().rstrip("\\") if os.path.exists(remote_url): # Local filepaths are allowed, but make sure we normalize them @@ -106,7 +110,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): logging.debug("Remote: " + remote_url.lower() + "\n") logging.debug("git_url: " + git_url.lower() + "\n") return False - except CalledProcessError as error: + except subprocess.CalledProcessError as error: logging.warn("Error obtaining git information. Error was: ") logging.warn(error) return False @@ -131,8 +135,8 @@ def get_git_info(repo): env['GIT_DIR'] = repo keys = ["GIT_DESCRIBE_TAG", "GIT_DESCRIBE_NUMBER", "GIT_DESCRIBE_HASH"] - process = Popen(["git", "describe", "--tags", "--long", "HEAD"], - stdout=PIPE, stderr=PIPE, + process = subprocess.Popen(["git", "describe", "--tags", "--long", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) output = process.communicate()[0].strip() output = output.decode('utf-8') @@ -142,8 +146,8 @@ def get_git_info(repo): d.update(dict(zip(keys, parts))) # get the _full_ hash of the current HEAD - process = Popen(["git", "rev-parse", "HEAD"], - stdout=PIPE, stderr=PIPE, env=env) + process = subprocess.Popen(["git", "rev-parse", "HEAD"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) output = process.communicate()[0].strip() output = output.decode('utf-8') @@ -256,6 +260,12 @@ def meta_vars(meta): if os.path.exists(git_url): # If git_url is a relative path instead of a url, convert it to an abspath git_url = normpath(join(meta.path, git_url)) + if sys.platform == 'win32': + cmd = "cygpath -w {0}".format(git_url) + if PY3: + git_url = subprocess.getoutput(cmd) + else: + git_url = subprocess.check_output(cmd.split()).rstrip().rstrip("\\") _x = False @@ -279,8 +289,8 @@ def get_cpu_count(): if sys.platform == "darwin": # multiprocessing.cpu_count() is not reliable on OSX # See issue #645 on github.com/conda/conda-build - out, err = Popen('sysctl -n hw.logicalcpu', shell=True, - stdout=PIPE).communicate() + out, err = subprocess.Popen('sysctl -n hw.logicalcpu', shell=True, + stdout=subprocess.PIPE).communicate() return out.decode('utf-8').strip() else: try: @@ -366,6 +376,70 @@ def system_vars(env_dict, prefix): return d +# http://code.activestate.com/recipes/576644-diff-two-dictionaries/#c9 +def _dict_diff(d1, d2): + """Shows entries that have changed or been added in d2 relative to d1""" + both = set(d1.keys()) & set(d2.keys()) + diff = {k: d2[k] for k in both if d1[k] != d2[k]} + diff.update({k: d2[k] for k in set(d2.keys()) - both}) + return diff + + +def _set_environ_from_subprocess_values(vars): + """ + Vars is an unprocessed string of envrionment variable output, such as from ```set``` + on Windows, or ```env``` elsewhere + """ + start_environ = os.environ + vars = vars.split("\n") + modified_environ = {var.split("=")[0].strip(): var.split("=")[1].strip() + for var in vars if "=" in var} + + # the diff is the only part we'll set for the actual build environment - mind you, + # the current process is not the actual build environment. That is always a native + # shell subprocess. + diff = _dict_diff(start_environ, modified_environ) + + # modify the current process with the activated/deactivated values + for key, value in modified_environ.items(): + os.environ[key] = value + + return diff + + +def activate_or_deactivate_env(action, env_name_or_path=""): + """ + Strategy is to open a subprocess, run activate, record the variables, + then apply them in our process + + action should be "activate" or "deactivate" + env_name_or_path should only be provided for activation. + """ + if sys.platform == "win32": + cmd = '"{0}\\Scripts\\{1}.bat" "{2}" && set'.format(sys.prefix, + action, + env_name_or_path) + if PY3: + # this method simplifies dealing with str vs bytestring on Py3. + vars = subprocess.getoutput(cmd).replace("\r\n", "\n") + else: + vars = subprocess.check_output(cmd).replace("\r\n", "\n") + else: + cmd = "source {0}/bin/{1} {2} && env".format(sys.prefix, action, env_name_or_path) + vars = subprocess.check_output(["bash", "-c", cmd], env=os.environ) + if PY3 and hasattr(vars, "decode"): + vars = vars.decode("UTF-8") + return _set_environ_from_subprocess_values(vars) + + +def activate_env(env_name_or_path): + return activate_or_deactivate_env("activate", env_name_or_path) + + +def deactivate_env(): + return activate_or_deactivate_env("deactivate") + + if __name__ == '__main__': e = get_dict() for k in sorted(e): diff --git a/conda_build/pypi.py b/conda_build/pypi.py index e93358d6e9..5ec478ed45 100644 --- a/conda_build/pypi.py +++ b/conda_build/pypi.py @@ -759,8 +759,9 @@ def run_setuppy(src_dir, temp_dir, python_version): # haywire. # TODO: Try with another version of Python if this one fails. Some # packages are Python 2 or Python 3 only. - create_env(config.build_prefix, ['python %s*' % python_version, 'pyyaml', - 'setuptools', 'numpy'], clear_cache=False) + create_env(config.build_prefix, ['python %s*' % python_version, + 'pyyaml', 'yaml', + 'setuptools', 'numpy'], clear_cache=False) stdlib_dir = join(config.build_prefix, 'Lib' if sys.platform == 'win32' else 'lib/python%s' % python_version) diff --git a/conda_build/windows.py b/conda_build/windows.py index df334d636c..750e75debe 100644 --- a/conda_build/windows.py +++ b/conda_build/windows.py @@ -186,10 +186,12 @@ def kill_processes(process_names=["msbuild.exe"]): continue -def build(m, bld_bat, dirty=False): - env = dict(os.environ) - env.update(environ.get_dict(m, dirty=dirty)) - env = environ.prepend_bin_path(env, config.build_prefix, True) +def build(m, bld_bat, dirty=False, env_diff=None): + if not env_diff: + env_diff = {} + env = environ.get_dict(m, dirty=dirty) + # env_diff contains activation info, among any other env variables set by activation scripts + env.update(env_diff) for name in 'BIN', 'INC', 'LIB': path = env['LIBRARY_' + name] diff --git a/setup.py b/setup.py index e6d3945455..5b684ca5ae 100755 --- a/setup.py +++ b/setup.py @@ -34,8 +34,8 @@ "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", ], description="tools for building conda packages", long_description=open('README.rst').read(), diff --git a/tests/test-recipes/metadata/binary_has_prefix_files/bld.bat b/tests/test-recipes/metadata/_binary_has_prefix_files/bld.bat similarity index 100% rename from tests/test-recipes/metadata/binary_has_prefix_files/bld.bat rename to tests/test-recipes/metadata/_binary_has_prefix_files/bld.bat diff --git a/tests/test-recipes/metadata/binary_has_prefix_files/build.sh b/tests/test-recipes/metadata/_binary_has_prefix_files/build.sh similarity index 100% rename from tests/test-recipes/metadata/binary_has_prefix_files/build.sh rename to tests/test-recipes/metadata/_binary_has_prefix_files/build.sh diff --git a/tests/test-recipes/metadata/binary_has_prefix_files/meta.yaml b/tests/test-recipes/metadata/_binary_has_prefix_files/meta.yaml similarity index 100% rename from tests/test-recipes/metadata/binary_has_prefix_files/meta.yaml rename to tests/test-recipes/metadata/_binary_has_prefix_files/meta.yaml diff --git a/tests/test-recipes/metadata/binary_has_prefix_files/run_test.py b/tests/test-recipes/metadata/_binary_has_prefix_files/run_test.py similarity index 100% rename from tests/test-recipes/metadata/binary_has_prefix_files/run_test.py rename to tests/test-recipes/metadata/_binary_has_prefix_files/run_test.py diff --git a/tests/test-recipes/metadata/binary_has_prefix_files/write_binary_has_prefix.py b/tests/test-recipes/metadata/_binary_has_prefix_files/write_binary_has_prefix.py similarity index 100% rename from tests/test-recipes/metadata/binary_has_prefix_files/write_binary_has_prefix.py rename to tests/test-recipes/metadata/_binary_has_prefix_files/write_binary_has_prefix.py diff --git a/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat new file mode 100644 index 0000000000..1a2f8ad626 --- /dev/null +++ b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/bld.bat @@ -0,0 +1,5 @@ +mkdir %PREFIX%\etc\conda\activate.d +echo set TEST_VAR=1 > %PREFIX%\etc\conda\activate.d\test.bat + +mkdir %PREFIX%\etc\conda\deactivate.d +echo set TEST_VAR= > %PREFIX%\etc\conda\deactivate.d\test.bat diff --git a/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/build.sh b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/build.sh new file mode 100644 index 0000000000..1c41766b8b --- /dev/null +++ b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/build.sh @@ -0,0 +1,7 @@ +mkdir -p $PREFIX/etc/conda/activate.d +echo "echo 'setting TEST_VAR' && export TEST_VAR=1" > $PREFIX/etc/conda/activate.d/test.sh +chmod +x $PREFIX/etc/conda/activate.d/test.sh + +mkdir -p $PREFIX/etc/conda/deactivate.d +echo "echo 'unsetting TEST_VAR' && unset TEST_VAR" > $PREFIX/etc/conda/deactivate.d/test.sh +chmod +x $PREFIX/etc/conda/deactivate.d/test.sh diff --git a/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/meta.yaml b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/meta.yaml new file mode 100644 index 0000000000..ef37fec222 --- /dev/null +++ b/tests/test-recipes/metadata/_conda-build-test-environment-vars-in-build-env/meta.yaml @@ -0,0 +1,6 @@ +package: + name: _conda-build-test-environment-vars-in-build-env + version: 1.0 + +about: + summary: test that scripts with activate.d and deactivate.d (set up activate/deactivate scripts) diff --git a/tests/test-recipes/metadata/has_prefix_files/run_test.py b/tests/test-recipes/metadata/has_prefix_files/run_test.py index cf0f861d04..fb50fba2a0 100644 --- a/tests/test-recipes/metadata/has_prefix_files/run_test.py +++ b/tests/test-recipes/metadata/has_prefix_files/run_test.py @@ -4,7 +4,7 @@ def main(): - prefix = os.environ['PREFIX'] + prefix = os.environ['PREFIX'].replace("\\", "/") with open(join(prefix, 'automatic-prefix')) as f: data = f.read() diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat b/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat new file mode 100644 index 0000000000..358ee87d83 --- /dev/null +++ b/tests/test-recipes/metadata/set_env_var_activate_build/bld.bat @@ -0,0 +1 @@ +if "%TEST_VAR%" == "" exit 1 \ No newline at end of file diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/build.sh b/tests/test-recipes/metadata/set_env_var_activate_build/build.sh new file mode 100644 index 0000000000..c709b3ecc8 --- /dev/null +++ b/tests/test-recipes/metadata/set_env_var_activate_build/build.sh @@ -0,0 +1,4 @@ +if [ -z "$TEST_VAR" ]; then + exit 1 +fi +exit 0 diff --git a/tests/test-recipes/metadata/set_env_var_activate_build/meta.yaml b/tests/test-recipes/metadata/set_env_var_activate_build/meta.yaml new file mode 100644 index 0000000000..00d57a8bc2 --- /dev/null +++ b/tests/test-recipes/metadata/set_env_var_activate_build/meta.yaml @@ -0,0 +1,11 @@ +package: + name: conda-build-test-environment-vars-in-build-env + version: 1.0 + +requirements: + build: + # having this as a build requirement should use the activate scripts that it contains + - _conda-build-test-environment-vars-in-build-env + +about: + summary: test that scripts with activate.d and deactivate.d provide environment variables appropriately diff --git a/tests/test-skeleton/test_skeleton.py b/tests/test-skeleton/test_skeleton.py index 7f7c5f27c6..00c84fa988 100644 --- a/tests/test-skeleton/test_skeleton.py +++ b/tests/test-skeleton/test_skeleton.py @@ -20,7 +20,7 @@ def fin(): def test_skeleton_by_name(tmpdir): - cmd = "conda skeleton pypi --output-dir {} conda".format(tmpdir) + cmd = "conda skeleton pypi --output-dir {} pip".format(tmpdir) subprocess.check_call(cmd.split()) diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index ed9d505f83..c8bfb22d07 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -23,25 +23,6 @@ def is_valid_dir(parent_dir, dirname): return valid -@pytest.mark.skipif(sys.platform != "win32", - reason="Problem only observed on Windows with win7 sdk") -def test_header_finding(): - """ - Windows sometimes very strangely cannot find headers in %LIBRARY_INC%. This has so far - only been a problem with the recipes that use the Win 7 SDK (python 3.4 builds) - """ - cmd = 'conda build --no-anaconda-upload {}/_pyyaml_find_header'.format(metadata_dir) - try: - output = subprocess.check_output(cmd.split()) - except subprocess.CalledProcessError as error: - print(error.output) - print(os.listdir(os.path.join(sys.prefix, "envs", "_build", "Library", "include"))) - raise - if PY3: - output = output.decode("UTF-8") - assert "forcing --without-libyaml" not in output - - # def test_CONDA_BLD_PATH(): # env = dict(os.environ) # cmd = 'conda build --no-anaconda-upload {}/source_git_jinja2'.format(metadata_dir) @@ -63,7 +44,15 @@ def test_output_build_path_git_source(): sys.version_info.major, sys.version_info.minor)) if PY3: output = output.decode("UTF-8") - assert output.rstrip() == test_path + error = error.decode("UTF-8") + assert output.rstrip() == test_path, error + + +@pytest.mark.skipif(sys.platform == "win32", + reason="no binary prefix manipulation done on windows.") +def test_binary_has_prefix_files(): + cmd = 'conda build --no-anaconda-upload {}/_binary_has_prefix_files'.format(metadata_dir) + subprocess.check_call(cmd.split()) @pytest.mark.skipif(sys.platform == "win32", @@ -95,6 +84,8 @@ def test_cached_source_not_interfere_with_versioning(): if PY3: output = output.decode("UTF-8") assert ("conda-build-test-source-git-jinja2-1.20.0" in output) + except: + raise finally: os.chdir(basedir)