diff --git a/refactor/README.md b/refactor/README.md index d0b00477..620d1487 100644 --- a/refactor/README.md +++ b/refactor/README.md @@ -1,28 +1,30 @@ -# Dockerfile generation for OpenVINO releases, for Ubuntu 20.04, Ubuntu 22.04, RHEL 8 +# Dockerfile templates for OpenVINO releases, for Ubuntu 20.04, Ubuntu 22.04 and RHEL 8 -## How to run +## How to use, how to run The tool does not require any non-standard Python packages, it only needs Python 3.10+ present -`configs/releases` directory contains configurations for all the supported releases. In general, if you want to build -`ubuntu20_dev:2024.3.0` image you should run this command: +`configs/releases` directory contains configurations for all the supported releases. For example, if you want to build +`ubuntu20_dev:2024.3.0` image, then you should run this command: ```bash python3 image.py 2024.3.0/ubuntu20 --preset dev --build ``` -This will generate `Dockerfile` and build it, tagging it `localhost/ubuntu20_dev:2024.3.0` +This will generate `Dockerfile`, build it and tag it `localhost/ubuntu20_dev:2024.3.0` + +If you add `--test` option, it will also run some tests associated with this image. ## Current support state Os support: * Ubuntu 20: ✅ * Ubuntu 22: ✅ -* Ubuntu 24: ❌ -* RHEL8: ❌ (TODO) +* Ubuntu 24: ❌ (WIP) +* RHEL8: ❌ (WIP) OpenVINO releases support: * before 2024.1.0 ❌ -* 2024.1.0 ❌(TODO) +* 2024.1.0 ❌(WIP) * 2024.2.0 ✅ * 2024.3.0 ✅ @@ -37,10 +39,22 @@ Note that even though `Intel NPU` is said to be supported it doesn't mean that e ## How to work with it +To make it easier to test your changes, whatever they are, you might find it useful to install `pytest` package, as it makes checking all images specified in this repository as easy as running `pytest` command in the root directory. It will also let you select or deselect some tests with pytest's `-k` option. Examples: + +This will only run dockerfile generation tests: +``` +pytest -v -k "generate" +``` + +This will run tests for all but "nightly" configs (to be precise: all the tests which names don't contain "nightly"): +``` +pytest -v -k "not nightly" +``` + ### When new OpenVINO release 1) Create a new release directory in `config/releases` called after the release version; -2) For each package build for specific os supported by this project, create a json file and use previous versions ad a template. +2) For each package build for specific os supported by this project, create a json file and use previous versions as a template. ### When new OS needs to be supported @@ -64,12 +78,12 @@ Note: recursion is forbidden, that is, the dependency graph must have no cycles. TODO: check for recursion, right now it will be infinitely loading if recursion appears. -#### Merging rules +#### Merging rules (a.k.a. what if there are values for the same key in different configs) 1) If either object is null (or if either is missing / is undefined) then the other is returned 2) If objects have different types then an error is returned 3) If objects are dictionaries then they are merged with this algorithm -4) otherwise the new object is returned instead of the old one (including lists) +4) otherwise the new object is returned instead of the old one (including lists!) TODO: ^^^ describe merging better ^^^ @@ -317,4 +331,3 @@ configuration would look like. - diff --git a/refactor/ci.py b/refactor/ci.py new file mode 100644 index 00000000..e69de29b diff --git a/refactor/configs/base/common.json b/refactor/configs/base/common.json index b0d84726..8a682e95 100644 --- a/refactor/configs/base/common.json +++ b/refactor/configs/base/common.json @@ -43,14 +43,16 @@ ], "extras": "caffe,kaldi,mxnet,onnx,pytorch,tensorflow,tensorflow2", "tests": [ - "check_omz_tools.sh" + "check_omz_tools.sh@CPU", + "check_omz_tools.sh@GPU" ] }, "preset_runtime": { "requires": [ "base", "device_cpu", - "device_gpu" + "device_gpu", + "device_npu" ] }, "preset_dev": { @@ -59,5 +61,13 @@ "openvino_dev" ] } + }, + "presets": { + "runtime": [ + "preset_runtime" + ], + "dev": [ + "preset_dev" + ] } } \ No newline at end of file diff --git a/refactor/configs/base/rhel8.json b/refactor/configs/base/rhel8.json new file mode 100644 index 00000000..07235c1c --- /dev/null +++ b/refactor/configs/base/rhel8.json @@ -0,0 +1,63 @@ +{ + "_based_on": "common", + "base_image": "registry.access.redhat.com/ubi8/ubi", + "os_id": "rhel8", + "components": { + "build_tools": { + "rpm": [ + "gcc", + "gcc-c++", + "make", + "cmake", + "pkgconfig" + ] + }, + "python": { + "rpm": [ + "python39", + "python3-virtualenv", + "python3-pip" + ], + "command": "python3.9" + }, + "intel-opencl-icd": { + "rpm": [ + "https://repositories.intel.com/graphics/rhel/8.6/intel-opencl-22.43.24595.35-i538.el8.x86_64.rpm" + ], + "requires": [ + "compute-runtime" + ] + }, + "level-zero": { + "rpm": [ + "https://repositories.intel.com/graphics/rhel/8.6/level-zero-1.8.8-i524.el8.x86_64.rpm" + ] + }, + "intel-level-zero-gpu": { + "rpm": [ + "https://repositories.intel.com/graphics/rhel/8.6/intel-level-zero-gpu-1.3.24595.35-i538.el8.x86_64.rpm" + ], + "requires": [ + "compute-runtime", + "level-zero" + ] + }, + "compute-runtime": { + "rpm": [ + "https://repositories.intel.com/graphics/rhel/8.6/intel-gmmlib-22.3.1-i529.el8.x86_64.rpm", + "https://repositories.intel.com/graphics/rhel/8.6/intel-igc-core-1.0.12504.6-i537.el8.x86_64.rpm", + "https://repositories.intel.com/graphics/rhel/8.6/intel-igc-opencl-1.0.12504.6-i537.el8.x86_64.rpm" + ] + }, + "opencl-loader": { + "rpm": [ + "https://vault.centos.org/8-stream/AppStream/x86_64/os/Packages/ocl-icd-2.2.12-1.el8.x86_64.rpm" + ] + }, + "device_npu": { + "requires": [], + "tests": [], + "_comment": "rhel8 doesn't support NPU, so we clear NPU requirements" + } + } +} \ No newline at end of file diff --git a/refactor/configs/base/ubuntu18.json b/refactor/configs/base/ubuntu18.json index e6530be4..6e8d12a8 100644 --- a/refactor/configs/base/ubuntu18.json +++ b/refactor/configs/base/ubuntu18.json @@ -53,6 +53,11 @@ "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.8708/intel-igc-core_1.0.8708_amd64.deb", "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.8708/intel-igc-opencl_1.0.8708_amd64.deb" ] + }, + "device_npu": { + "requires": [], + "tests": [], + "_comment": "ubuntu20 doesn't support NPU, so we clear NPU requirements" } } } \ No newline at end of file diff --git a/refactor/configs/base/ubuntu20.json b/refactor/configs/base/ubuntu20.json index f4d4aa31..9498b0ff 100644 --- a/refactor/configs/base/ubuntu20.json +++ b/refactor/configs/base/ubuntu20.json @@ -32,6 +32,11 @@ "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.12037.1/intel-igc-core_1.0.12037.1_amd64.deb", "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.12037.1/intel-igc-opencl_1.0.12037.1_amd64.deb" ] + }, + "device_npu": { + "requires": [], + "tests": [], + "_comment": "ubuntu20 doesn't support NPU, so we clear NPU requirements" } } } \ No newline at end of file diff --git a/refactor/configs/releases/2023.3/ubuntu18.json b/refactor/configs/releases/2023.3/ubuntu18.json index ccfd5687..ea3db4a5 100644 --- a/refactor/configs/releases/2023.3/ubuntu18.json +++ b/refactor/configs/releases/2023.3/ubuntu18.json @@ -1,16 +1,8 @@ { "_based_on": "ubuntu18", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu18_2023.3.0.13775.ceeafaf64f3_x86_64.tgz", "version": "2023.3.0" - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] } } diff --git a/refactor/configs/releases/2023.3/ubuntu20.json b/refactor/configs/releases/2023.3/ubuntu20.json index 23d1a9ab..953c023d 100644 --- a/refactor/configs/releases/2023.3/ubuntu20.json +++ b/refactor/configs/releases/2023.3/ubuntu20.json @@ -1,16 +1,8 @@ { "_based_on": "ubuntu20", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu20_2023.3.0.13775.ceeafaf64f3_x86_64.tgz", "version": "2023.3.0" - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] } } diff --git a/refactor/configs/releases/2023.3/ubuntu22.json b/refactor/configs/releases/2023.3/ubuntu22.json index a807ebf7..d01de8b0 100644 --- a/refactor/configs/releases/2023.3/ubuntu22.json +++ b/refactor/configs/releases/2023.3/ubuntu22.json @@ -1,16 +1,15 @@ { "_based_on": "ubuntu22", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu22_2023.3.0.13775.ceeafaf64f3_x86_64.tgz", "version": "2023.3.0" }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] + "components": { + "device_npu": { + "requires": [], + "tests": [], + "_comment": "2023* releases don't support NPU, so we clear NPU requirements" + } } } diff --git a/refactor/configs/releases/2024.1/ubuntu20.json b/refactor/configs/releases/2024.1/ubuntu20.json index 4b228c7e..5628bf79 100644 --- a/refactor/configs/releases/2024.1/ubuntu20.json +++ b/refactor/configs/releases/2024.1/ubuntu20.json @@ -1,16 +1,8 @@ { "_based_on": "ubuntu20", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64.tgz", "version": "2024.1.0" - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] } } diff --git a/refactor/configs/releases/2024.1/ubuntu22.json b/refactor/configs/releases/2024.1/ubuntu22.json index 26aa2449..b1db98be 100644 --- a/refactor/configs/releases/2024.1/ubuntu22.json +++ b/refactor/configs/releases/2024.1/ubuntu22.json @@ -1,18 +1,8 @@ { "_based_on": "ubuntu22", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz", "version": "2024.1.0" - }, - "presets": { - "runtime": [ - "preset_runtime", - "device_npu" - ], - "dev": [ - "preset_dev", - "device_npu" - ] } } diff --git a/refactor/configs/releases/2024.2/ubuntu20.json b/refactor/configs/releases/2024.2/ubuntu20.json index fa5834c6..64e06f27 100644 --- a/refactor/configs/releases/2024.2/ubuntu20.json +++ b/refactor/configs/releases/2024.2/ubuntu20.json @@ -1,16 +1,8 @@ { "_based_on": "ubuntu20", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64.tgz", "version": "2024.2.0" - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] } } \ No newline at end of file diff --git a/refactor/configs/releases/2024.2/ubuntu22.json b/refactor/configs/releases/2024.2/ubuntu22.json index 4066e834..57b84b52 100644 --- a/refactor/configs/releases/2024.2/ubuntu22.json +++ b/refactor/configs/releases/2024.2/ubuntu22.json @@ -1,18 +1,8 @@ { "_based_on": "ubuntu22", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64.tgz", "version": "2024.2.0" - }, - "presets": { - "runtime": [ - "preset_runtime", - "device_npu" - ], - "dev": [ - "preset_dev", - "device_npu" - ] } } \ No newline at end of file diff --git a/refactor/configs/releases/2024.3/rhel8.json b/refactor/configs/releases/2024.3/rhel8.json new file mode 100644 index 00000000..11838f27 --- /dev/null +++ b/refactor/configs/releases/2024.3/rhel8.json @@ -0,0 +1,8 @@ +{ + "_based_on": "rhel8", + "_template": "Dockerfile_default.j2", + "package": { + "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_rhel8_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz", + "version": "2024.3.0" + } +} diff --git a/refactor/configs/releases/2024.3/ubuntu20.json b/refactor/configs/releases/2024.3/ubuntu20.json index 930c9307..569eaf9c 100644 --- a/refactor/configs/releases/2024.3/ubuntu20.json +++ b/refactor/configs/releases/2024.3/ubuntu20.json @@ -1,16 +1,8 @@ { "_based_on": "ubuntu20", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu20_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz", "version": "2024.3.0" - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] } } \ No newline at end of file diff --git a/refactor/configs/releases/2024.3/ubuntu22.json b/refactor/configs/releases/2024.3/ubuntu22.json index 31671523..4b8ff47a 100644 --- a/refactor/configs/releases/2024.3/ubuntu22.json +++ b/refactor/configs/releases/2024.3/ubuntu22.json @@ -1,18 +1,8 @@ { "_based_on": "ubuntu22", - "_template": "Dockerfile_2024_deb_default.j2", + "_template": "Dockerfile_default.j2", "package": { "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz", "version": "2024.3.0" - }, - "presets": { - "runtime": [ - "preset_runtime", - "device_npu" - ], - "dev": [ - "preset_dev", - "device_npu" - ] } } \ No newline at end of file diff --git a/refactor/configs/releases/2024.4.0-nightly/ubuntu20.json b/refactor/configs/releases/2024.4.0-nightly/ubuntu20.json deleted file mode 100644 index ca2b8c3d..00000000 --- a/refactor/configs/releases/2024.4.0-nightly/ubuntu20.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "_based_on": "ubuntu20", - "_template": "Dockerfile_2024_deb_default.j2", - "package": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/closed_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/swf_drop/packages/releases/l_openvino_toolkit_ubuntu20_2024.4.0.16333.70aa39a3ab1_x86_64.tgz", - "version": "2024.4.0", - "wheels": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/private_linux_manylinux2014_release/wheels/", - "version": "2024.4.0.dev20240812" - } - }, - "presets": { - "runtime": [ - "preset_runtime" - ], - "dev": [ - "preset_dev" - ] - } -} \ No newline at end of file diff --git a/refactor/configs/releases/2024.4.0-nightly/ubuntu22.json b/refactor/configs/releases/2024.4.0-nightly/ubuntu22.json deleted file mode 100644 index 7f5f0a8a..00000000 --- a/refactor/configs/releases/2024.4.0-nightly/ubuntu22.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "_based_on": "ubuntu22", - "_template": "Dockerfile_2024_deb_default.j2", - "package": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/closed_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/swf_drop/packages/releases/l_openvino_toolkit_ubuntu22_2024.4.0.16333.70aa39a3ab1_x86_64.tgz", - "version": "2024.4.0", - "wheels": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/private_linux_manylinux2014_release/wheels/", - "version": "2024.4.0.dev20240812" - } - }, - "presets": { - "runtime": [ - "preset_runtime", - "device_npu" - ], - "dev": [ - "preset_dev", - "device_npu" - ] - } -} \ No newline at end of file diff --git a/refactor/configs/releases/2024.4.0-nightly/ubuntu24.json b/refactor/configs/releases/2024.4.0-nightly/ubuntu24.json deleted file mode 100644 index 0468f4cc..00000000 --- a/refactor/configs/releases/2024.4.0-nightly/ubuntu24.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "_based_on": "ubuntu24", - "_template": "Dockerfile_2024_deb_default.j2", - "package": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/closed_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/swf_drop/packages/releases/l_openvino_toolkit_ubuntu24_2024.4.0.16333.70aa39a3ab1_x86_64.tgz", - "version": "2024.4.0", - "wheels": { - "url": "http://ov-share-03.sclab.intel.com/openvino_ci/private_builds/dldt/master/commit/70aa39a3ab1c8554fe1bcd1dde13ce2e1ad19a70/private_linux_manylinux2014_release/wheels/", - "version": "2024.4.0.dev20240812" - } - }, - "presets": { - "runtime": [ - "preset_runtime", - "device_npu" - ], - "dev": [ - "preset_dev", - "device_npu" - ] - } -} \ No newline at end of file diff --git a/refactor/generate.py b/refactor/generate.py new file mode 100644 index 00000000..c3fc829d --- /dev/null +++ b/refactor/generate.py @@ -0,0 +1,67 @@ +import subprocess +import argparse +import json +import os + +from jinja2 import Environment, FileSystemLoader, ChainableUndefined + +from lib import config, package_filename + + +jinjaenv = Environment(loader=FileSystemLoader("templates"), undefined=ChainableUndefined) + +parser = argparse.ArgumentParser() +parser.add_argument("--config", help="Use pre-existing config to generate Dockerfile.") +parser.add_argument("--package_url", "--package-url", help="Generate config for given package url.") +parser.add_argument("--wheels_url", "--wheels-url", help="Also specify wheels url when generating the config.") +parser.add_argument("-p", "--preset", default="runtime", help="preset to build, e.g. 'dev' or 'runtime'") +parser.add_argument("--include-components", + help="list of components to forcefully enable (separated by comma)") +parser.add_argument("--exclude-components", help="list of components to forcefully disable " + "(separated by comma), Note: this option breaks component requirements") +parser.add_argument("-o", "--out", "--output", default="Dockerfile", + help="output file name to write Dockerfile to, defaults to 'Dockerfile'") + + +args = parser.parse_args() + +if args.config and args.package_url: + raise Exception("Mutually exclusive options: package_url and config") + +if not args.config and not args.package_url: + raise Exception("Either package_url or config must be specified") + + +if args.config: + context = config.default_env.load(args.config) +else: + package_info = package_filename.parse(args.package_url.rsplit("/", 1)[-1]) + config_data = { + "_based_on": package_info["os"], + "_template": "Dockerfile_default.j2", + "package": { + "url": args.package_url, + "version": package_info.get("version"), + "wheels": { + "url": args.wheels_url, + # "version": None + } + } + } + context = config.default_env.from_dict(config_data) + +include_components = args.include_components.split(",") if args.include_components else () +exclude_components = args.exclude_components.split(",") if args.exclude_components else () + +context = config.process( + context, + args.preset, + include_components=include_components, + exclude_components=exclude_components +) + +template = context["_template"] +dockerfile = jinjaenv.get_template(template).render(context) + +with open(args.out, "w") as outfile: + outfile.write(dockerfile) diff --git a/refactor/image.py b/refactor/image.py deleted file mode 100644 index 97518dce..00000000 --- a/refactor/image.py +++ /dev/null @@ -1,67 +0,0 @@ -import subprocess -import argparse - -from jinja2 import Environment, FileSystemLoader, ChainableUndefined - -from lib import config - - -jinjaenv = Environment(loader=FileSystemLoader("templates"), undefined=ChainableUndefined) -confenv = config.Env("configs/releases", "configs/base") - -parser = argparse.ArgumentParser() -parser.add_argument("config", help="release config to generate for, e.g. '2024.3.0/ubuntu20'") -parser.add_argument("-p", "-d", "--preset", default="runtime", help="preset to build, e.g. 'dev' or 'runtime'") -parser.add_argument("--include-components", - help="list of components to forcefully enable (separated by comma)") -parser.add_argument("--exclude-components", help="list of components to forcefully disable " - "(separated by comma), Note: this option breaks component requirements") -parser.add_argument("-o", "--out", "--output", default="Dockerfile", - help="output file name to write Dockerfile to, defaults to 'Dockerfile'") -parser.add_argument("-b", "--build", action="store_true", help="also build the image and tag it with '{os}_{preset}:{version}'") -parser.add_argument("--test", action="store_true", help="also test the built image (implies --build option)") - -args = parser.parse_args() - -context = confenv.load(args.config) - -include_components = args.include_components.split(",") if args.include_components else () -exclude_components = args.exclude_components.split(",") if args.exclude_components else () - -context = config.process( - context, - args.preset, - include_components=include_components, - exclude_components=exclude_components -) - -template = context["_template"] -dockerfile = jinjaenv.get_template(template).render(context) - -with open(args.out, "w") as outfile: - outfile.write(dockerfile) - -if args.build or args.test: - os_id = context["os_id"] - package_version = context["package"]["version"] - image_tag = f"localhost/{os_id}_{args.preset}:{package_version}" - subprocess.run(["docker", "build", "-t", image_tag, "-f", args.out]) - -if args.test: - for test_file in context["tests"]: - print(f"Running {test_file} ...") - arg = "" - if "@" in test_file: - test_file, arg = test_file.split("@", 1) - result = subprocess.run([ - "docker", "run", "--rm", - "-v", "./tests:/tests", - "--device", "/dev/dri", - # "--device", "/dev/accel", - image_tag, - "bash", "-c", f"/tests/{test_file} {arg}" - ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if result.returncode != 0: - print("FAILED") - else: - print("PASSED") diff --git a/refactor/lib/config.py b/refactor/lib/config.py index 3d06917e..34a25ab4 100644 --- a/refactor/lib/config.py +++ b/refactor/lib/config.py @@ -66,6 +66,9 @@ def load(self, product): # TODO: if valid path is given instead of a product, use it with open(f"{self.product_dir}/{product}.json") as prodconf: return self._load(json.load(prodconf)) + + def from_dict(self, data): + return self._load(data) def discover(self): for product_name in os.scandir(self.product_dir): @@ -76,6 +79,9 @@ def discover(self): continue yield f"{product_name.name}/{product_conf.name.rsplit('.', 1)[0]}" +CONFIGS_ROOT = os.path.realpath(f"{__file__}/../../configs") +default_env = Env(f"{CONFIGS_ROOT}/releases", f"{CONFIGS_ROOT}/base") + def process(config, preset, include_components=(), exclude_components=()): # enable --include-components @@ -110,8 +116,7 @@ def enable_recursively(components, cname): apt_packages = [] apt_downloads = [] for cname, component in config["components"].items(): - enabled = component.get("enable") - if not enabled: + if not component.get("enable"): continue apt = component.get("apt") if not apt: @@ -130,6 +135,24 @@ def enable_recursively(components, cname): "download_dir": "/tmp/apt_dl" } + # collect rpm packages + rpm_packages = [] + for cname, component in config["components"].items(): + if not component.get("enable"): + continue + rpm = component.get("rpm") + if not rpm: + continue + rpm_packages.extend(rpm) + + config["rpm"] = { + "packages": rpm_packages + } + + assert bool(config["rpm"]["packages"]) ^ \ + bool(config["apt"]["packages"] or config["apt"]["downloads"]), \ + "Can't be both rpm and apt based system. Something's wrong" + # collect tests tests = set() for cname, component in config["components"].items(): diff --git a/refactor/lib/package_filename.py b/refactor/lib/package_filename.py new file mode 100644 index 00000000..b4b22fa6 --- /dev/null +++ b/refactor/lib/package_filename.py @@ -0,0 +1,31 @@ +import re + +PACKAGE_FILENAME_RE = re.compile( + r"^([lmw])?_?openvino_(toolkit|genai|tokenizers)" + r"_?(dev|data_dev|runtime|ie_runtime)?" + r"_(centos\d+|debian\d+|raspbian|rhel\d+|ubuntu\d+|macos_\d+_\d+|osx|windows|)" + r"_?(arm|p|dev)?" + r"_(20\d\d\.\d\.\d+).?([\.\da-z]+)?" + r"_?(fpga_only|pot)?" + r"_?(x86_64|armhf|arm64)?" + r"\.(tgz|tar\.gz|zip)$" +) + +FIELDS = ( + "os_fam_hint", # l, w, m or None + "dist", # toolkit, genai or tokenizers + "_subdist", # deprecated; dev, data_dev, runtime or ie_runtime + "os", # centos*, debian*, raspbian, rhel*, ubuntu*, macos_*_*, osx, windows or "" + "_prefix", # deprecated; arm, p or dev + "version", # main version: three numbers + "version_extra", # extra version, no specific format + "_version_suffix", # deprecated; fpga_only or pot + "arch", # x86_64, armhf, arm64 or None + "ext", # tgz, tar.gz or zip +) + +def parse(filename) -> dict: + match = PACKAGE_FILENAME_RE.match(filename) + if not match: + return None + return dict(zip(FIELDS, match.groups())) diff --git a/refactor/templates/Dockerfile_2024_deb_default.j2 b/refactor/templates/Dockerfile_default.j2 similarity index 91% rename from refactor/templates/Dockerfile_2024_deb_default.j2 rename to refactor/templates/Dockerfile_default.j2 index da1bb23a..e4f719d2 100644 --- a/refactor/templates/Dockerfile_2024_deb_default.j2 +++ b/refactor/templates/Dockerfile_default.j2 @@ -20,11 +20,14 @@ #} FROM {{ base_image }} -ARG DEBIAN_FRONTEND=noninteractive # System package dependencies +{%- if apt.packages or apt.downloads %} +ARG DEBIAN_FRONTEND=noninteractive RUN apt update && \ + {% if apt.packages -%} apt install -y --no-install-recommends {{ apt.packages | join(" ") }} && \ + {% endif -%} {% if apt.downloads -%} mkdir -p {{ apt.download_dir }} && \ cd {{ apt.download_dir }} && \ @@ -33,6 +36,13 @@ RUN apt update && \ apt install -y --no-install-recommends {{ apt.downloads | map(attribute=1) | join(" ") }} && \ {% endif -%} rm -rf {{ apt.download_dir }} /var/lib/apt/lists/* +{%- endif %} + +{%- if rpm.packages %} +RUN yum install -y {{ rpm.packages | join(" ") }} && \ + yum clean all && \ + rm -rf /var/cache/yum +{%- endif %} # User setup (is it needed?) RUN sed -si "s/^UMASK.+/UMASK 000/g" /etc/login.defs diff --git a/refactor/tests/__init__.py b/refactor/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/refactor/tests/check_omz_tools.sh b/refactor/tests/in_container/check_omz_tools.sh similarity index 73% rename from refactor/tests/check_omz_tools.sh rename to refactor/tests/in_container/check_omz_tools.sh index 6478a967..8684baaa 100755 --- a/refactor/tests/check_omz_tools.sh +++ b/refactor/tests/in_container/check_omz_tools.sh @@ -6,4 +6,4 @@ omz_downloader --name yolo-v1-tiny-tf benchmark_app -m public/yolo-v1-tiny-tf/yolo-v1-tiny.pb -shape [1,416,416,3] -t 1 ovc public/yolo-v1-tiny-tf/yolo-v1-tiny.pb -benchmark_app -m yolo-v1-tiny.xml -shape [1,416,416,3] -t 1 +benchmark_app -m yolo-v1-tiny.xml -shape [1,416,416,3] -t 1 -d $1 diff --git a/refactor/tests/check_python_openvino.sh b/refactor/tests/in_container/check_python_openvino.sh similarity index 100% rename from refactor/tests/check_python_openvino.sh rename to refactor/tests/in_container/check_python_openvino.sh diff --git a/refactor/tests/in_container/local_library_loadable.sh b/refactor/tests/in_container/local_library_loadable.sh new file mode 100755 index 00000000..10acb9de --- /dev/null +++ b/refactor/tests/in_container/local_library_loadable.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -e +ldd $(find -name $1) 2>/dev/null +ldd $(find -name $1) 2>/dev/null | grep "not found" && exit 1 || exit 0 diff --git a/refactor/tests/local_library_loadable.sh b/refactor/tests/local_library_loadable.sh deleted file mode 100755 index 89ccf858..00000000 --- a/refactor/tests/local_library_loadable.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -ldd $(find -name $1) | grep "not found" && exit 1 || exit 0 diff --git a/refactor/tests/test_imagepy.py b/refactor/tests/test_imagepy.py new file mode 100644 index 00000000..0d68f33b --- /dev/null +++ b/refactor/tests/test_imagepy.py @@ -0,0 +1,77 @@ +import subprocess +import tempfile +import uuid +import sys +import os + +import pytest + +from lib.config import default_env + + +def test_runs_without_virtualenv(): + # the idea is that the script runs without any extra modules + # the best way that came to my mind + + environ = os.environ.copy() + if "VIRTUAL_ENV" in environ: + del environ["VIRTUAL_ENV"] + subprocess.check_call( + args=[os.path.realpath(sys.executable), "image.py", "--help"], + env=environ + ) + + +@pytest.fixture(scope="session") +def temp_dir(): + dir = tempfile.TemporaryDirectory(prefix="pytest_") + yield dir.name + dir.cleanup() + del dir + + +@pytest.fixture +def temp_file(temp_dir): + files = [] + def new_file(name): + new_name = temp_dir + "/" + name + "_" + uuid.uuid4().hex[:10] + files.append(new_name) + return new_name + yield new_file + for file in files: + os.remove(file) + + +def parametrize_config_preset(): + for conf in default_env.discover(): + conf_data = default_env.load(conf) + if not "presets" in conf_data: + continue + for preset in conf_data["presets"]: + yield (conf, preset) + return [] + + +@pytest.mark.parametrize('config,preset', parametrize_config_preset()) +def test_dockerfile_generate(config, preset, temp_file): + dockerfile = temp_file("Dockerfile") + subprocess.check_call([sys.executable, "image.py", config, "-p", preset, "-o", dockerfile]) + + +@pytest.mark.parametrize('config,preset', parametrize_config_preset()) +def test_dockerfile_build_internal(config, preset, temp_file): + dockerfile = temp_file("Dockerfile") + subprocess.check_call([sys.executable, "image.py", config, "-p", preset, "-o", dockerfile, "--build"]) + + +@pytest.mark.parametrize('config,preset', parametrize_config_preset()) +def test_dockerfile_build_manually(config, preset, temp_file): + dockerfile = temp_file("Dockerfile") + subprocess.check_call([sys.executable, "image.py", config, "-p", preset, "-o", dockerfile]) + subprocess.check_call(["docker", "build", "-f", dockerfile]) + + +@pytest.mark.parametrize('config,preset', parametrize_config_preset()) +def test_dockerfile_test_internal(config, preset, temp_file): + dockerfile = temp_file("Dockerfile") + subprocess.check_call([sys.executable, "image.py", config, "-p", preset, "-o", dockerfile, "--build", "--test"])