diff --git a/appveyor/windows_templates/build_windows_templates.py b/appveyor/windows_templates/build_windows_templates.py index 456ae19f3..2e6b910ea 100644 --- a/appveyor/windows_templates/build_windows_templates.py +++ b/appveyor/windows_templates/build_windows_templates.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -#!/usr/bin/env python """Script to build windows templates.""" import argparse @@ -11,54 +10,67 @@ import subprocess import sys import time - -from typing import Callable +from typing import Callable, Optional parser = argparse.ArgumentParser(description="Build windows templates.") parser.add_argument( - "--build_dir", default=r"C:\grrbuild", help="GRR build directory.") + "--build_dir", default=r"C:\grrbuild", help="GRR build directory." +) parser.add_argument( "--grr_src", default=r"C:\grrbuild\grr", - help="Location of the grr src code. If it doesn't exist " - " at this path we'll try to check it out from github.") + help=( + "Location of the grr src code. If it doesn't exist " + " at this path we'll try to check it out from github." + ), +) parser.add_argument( "--output_dir", default=r"C:\grrbuild\output", - help="Destination directory for the templates.") + help="Destination directory for the templates.", +) parser.add_argument( "--test_repack_install", action="store_true", default=False, - help="Test repacking by calling repack on the template after building," - "then try and install the result. For use by integration tests. If you use " - "this option you must run as admin.") + help=( + "Test repacking by calling repack on the template after building,then" + " try and install the result. For use by integration tests. If you use" + " this option you must run as admin." + ), +) parser.add_argument( "--wheel_dir", default=None, - help="A directory that will be passed to pip as the wheel-dir parameter.") + help="A directory that will be passed to pip as the wheel-dir parameter.", +) parser.add_argument( "--expect_service_running", dest="expect_service_running", action="store_true", - help="Triggers whether after installation the GRR service should be " - "running or not. Used for testing the installation.") + help=( + "Triggers whether after installation the GRR service should be " + "running or not. Used for testing the installation." + ), +) parser.add_argument( "--noexpect_service_running", dest="expect_service_running", - action="store_false") + action="store_false", +) parser.set_defaults(expect_service_running=True) parser.add_argument( "--config", default="", - help="Path to the config file to be used when building templates.") + help="Path to the config file to be used when building templates.", +) args = parser.parse_args() @@ -79,13 +91,16 @@ def _FileRetryLoop(path: str, f: Callable[[], None]) -> None: return except OSError as e: attempts += 1 - if (e.errno == errno.EACCES and - attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS): + if ( + e.errno == errno.EACCES + and attempts < _FILE_RETRY_LOOP_RETRY_TIME_SECS + ): # The currently installed GRR process may stick around for a few # seconds after the service is terminated (keeping the contents of # the installation directory locked). - logging.info("Permission-denied error while trying to process %s.", - path) + logging.info( + "Permission-denied error while trying to process %s.", path + ) time.sleep(1) else: raise @@ -99,27 +114,6 @@ def _Rename(src: str, dst: str) -> None: _FileRetryLoop(src, lambda: os.rename(src, dst)) -def _RmTreePseudoTransactional(path: str) -> None: - """Removes `path`. - - Makes sure that either `path` is gone or that it is still present as - it was. - - Args: - path: The path to remove. - """ - temp_path = f"{path}_orphaned_{int(time.time())}" - logging.info("Trying to rename %s -> %s.", path, temp_path) - - _Rename(path, temp_path) - - try: - logging.info("Trying to remove %s.", temp_path) - _RmTree(temp_path) - except: # pylint: disable=bare-except - logging.info("Failed to remove %s. Ignoring.", temp_path, exc_info=True) - - def _VerboseCheckCall(params): logging.info("Running: %s", params) @@ -141,8 +135,9 @@ def SetupVars(self): self.virtualenv64 = os.path.join(args.build_dir, "python_64") self.grr_client_build64 = "grr_client_build" - self.virtualenv_python64 = os.path.join(self.virtualenv64, - r"Scripts\python.exe") + self.virtualenv_python64 = os.path.join( + self.virtualenv64, r"Scripts\python.exe" + ) self.git = r"git" @@ -180,55 +175,72 @@ def Clean(self): def GitCheckoutGRR(self): os.chdir(args.build_dir) subprocess.check_call( - [self.git, "clone", "https://github.com/google/grr.git"]) + [self.git, "clone", "https://github.com/google/grr.git"] + ) def MakeProtoSdist(self): os.chdir(os.path.join(args.grr_src, "grr/proto")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_proto-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "proto", "requirements", "windows.txt") def MakeCoreSdist(self): os.chdir(os.path.join(args.grr_src, "grr/core")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir, "--no-sync-artifacts" + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, + "--no-sync-artifacts", ]) return glob.glob( os.path.join(args.build_dir, "grr_response_core-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "core", "requirements", "windows.txt") def MakeClientSdist(self): os.chdir(os.path.join(args.grr_src, "grr/client/")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_client-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "client", "requirements", "windows.txt") def MakeClientBuilderSdist(self): os.chdir(os.path.join(args.grr_src, "grr/client_builder/")) subprocess.check_call([ - self.virtualenv_python64, "setup.py", "sdist", "--formats=zip", - "--dist-dir=%s" % args.build_dir + self.virtualenv_python64, + "setup.py", + "sdist", + "--formats=zip", + "--dist-dir=%s" % args.build_dir, ]) return glob.glob( os.path.join(args.build_dir, "grr_response_client_builder-*.zip") - ).pop() + ).pop(), os.path.join(args.grr_src, "grr", "client_builder", "requirements", "windows.txt") - def InstallGRR(self, path): + def InstallGRR(self, path: str, requirements_file: Optional[str] = None): """Installs GRR.""" + if requirements_file: + install_requirements_cmd = ["pip", "intall", "--require-hashes", requirements_file] + subprocess.check_call(install_requirements_cmd) - cmd64 = ["pip", "install"] + cmd64 = ["pip", "install", "--no-deps", "--no-index"] if args.wheel_dir: - cmd64 += ["--no-index", r"--find-links=file:///%s" % args.wheel_dir] + cmd64 += [r"--find-links=file:///%s" % args.wheel_dir] cmd64.append(path) @@ -242,8 +254,12 @@ def BuildTemplates(self): """ if args.config: build_args = [ - "--verbose", "--config", args.config, "build", "--output", - args.output_dir + "--verbose", + "--config", + args.config, + "build", + "--output", + args.output_dir, ] else: build_args = ["--verbose", "build", "--output", args.output_dir] @@ -268,9 +284,11 @@ def _WixToolsPath(self) -> str: def _RepackTemplates(self): """Repack templates with a dummy config.""" dummy_config = os.path.join( - args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml") - template_amd64 = glob.glob(os.path.join(args.output_dir, - "*_amd64*.zip")).pop() + args.grr_src, "grr/test/grr_response_test/test_data/dummyconfig.yaml" + ) + template_amd64 = glob.glob( + os.path.join(args.output_dir, "*_amd64*.zip") + ).pop() fleetspeak_config = os.path.join( args.grr_src, @@ -345,8 +363,9 @@ def _CheckInstallSuccess(self): raise RuntimeError("Install failed, no files at: %s" % self.install_path) try: - output = subprocess.check_output(["sc", "query", self.service_name], - encoding="utf-8") + output = subprocess.check_output( + ["sc", "query", self.service_name], encoding="utf-8" + ) service_running = "RUNNING" in output except subprocess.CalledProcessError as e: output = e.output @@ -370,13 +389,15 @@ def _CheckInstallSuccess(self): if self.expect_service_running: if not service_running: raise RuntimeError( - "GRR service not running after install, sc query output: %s" % - output) + "GRR service not running after install, sc query output: %s" + % output + ) else: if service_running: raise RuntimeError( "GRR service running after install with expect_service_running == " - "False, sc query output: %s" % output) + "False, sc query output: %s" % output + ) def _InstallInstallers(self): """Install the installer built by RepackTemplates.""" @@ -404,15 +425,15 @@ def Build(self): if not os.path.exists(args.grr_src): self.GitCheckoutGRR() - proto_sdist = self.MakeProtoSdist() - core_sdist = self.MakeCoreSdist() - client_sdist = self.MakeClientSdist() - client_builder_sdist = self.MakeClientBuilderSdist() - - self.InstallGRR(proto_sdist) - self.InstallGRR(core_sdist) - self.InstallGRR(client_sdist) - self.InstallGRR(client_builder_sdist) + proto_sdist, proto_requirements = self.MakeProtoSdist() + core_sdist, core_requirements = self.MakeCoreSdist() + client_sdist, client_requirements = self.MakeClientSdist() + client_builder_sdist, client_builder_requirements = self.MakeClientBuilderSdist() + + self.InstallGRR(proto_sdist, proto_requirements) + self.InstallGRR(core_sdist, core_requirements) + self.InstallGRR(client_sdist, client_requirements) + self.InstallGRR(client_builder_sdist, client_builder_requirements) self.BuildTemplates() if args.test_repack_install: self._RepackTemplates()