wip

openvinotoolkit · Sep 10, 2024 · 8194aaf · 8194aaf
1 parent edc9179
commit 8194aaf
Show file tree

Hide file tree

Showing 30 changed files with 350 additions and 238 deletions.
diff --git a/refactor/README.md b/refactor/README.md
@@ -1,28 +1,30 @@
-# Dockerfile generation for OpenVINO releases, for Ubuntu 20.04, Ubuntu 22.04, RHEL 8
+# Dockerfile templates for OpenVINO releases, for Ubuntu 20.04, Ubuntu 22.04 and RHEL 8
 
-## How to run
+## How to use, how to run
 
 The tool does not require any non-standard Python packages, it only needs Python 3.10+ present
 
-`configs/releases` directory contains configurations for all the supported releases. In general, if you want to build
-`ubuntu20_dev:2024.3.0` image you should run this command:
+`configs/releases` directory contains configurations for all the supported releases. For example, if you want to build 
+`ubuntu20_dev:2024.3.0` image, then you should run this command:
 
 ```bash
 python3 image.py 2024.3.0/ubuntu20 --preset dev --build
 ```
-This will generate `Dockerfile` and build it, tagging it `localhost/ubuntu20_dev:2024.3.0`
+This will generate `Dockerfile`, build it and tag it `localhost/ubuntu20_dev:2024.3.0`
+
+If you add `--test` option, it will also run some tests associated with this image.
 
 ## Current support state
 
 Os support:
 * Ubuntu 20: ✅
 * Ubuntu 22: ✅
-* Ubuntu 24: ❌
-* RHEL8: ❌ (TODO)
+* Ubuntu 24: ❌ (WIP)
+* RHEL8: ❌ (WIP)
 
 OpenVINO releases support:
 * before 2024.1.0 ❌
-* 2024.1.0 ❌(TODO)
+* 2024.1.0 ❌(WIP)
 * 2024.2.0 ✅
 * 2024.3.0 ✅
 
@@ -37,10 +39,22 @@ Note that even though `Intel NPU` is said to be supported it doesn't mean that e
 
 ## How to work with it
 
+To make it easier to test your changes, whatever they are, you might find it useful to install `pytest` package, as it makes checking all images specified in this repository as easy as running `pytest` command in the root directory. It will also let you select or deselect some tests with pytest's `-k` option. Examples:
+
+This will only run dockerfile generation tests:
+```
+pytest -v -k "generate"
+```
+
+This will run tests for all but "nightly" configs (to be precise: all the tests which names don't contain "nightly"):
+```
+pytest -v -k "not nightly"
+```
+
 ### When new OpenVINO release
 
 1) Create a new release directory in `config/releases` called after the release version;
-2) For each package build for specific os supported by this project, create a json file and use previous versions ad a template.
+2) For each package build for specific os supported by this project, create a json file and use previous versions as a template.
 
 ### When new OS needs to be supported
 
@@ -64,12 +78,12 @@ Note: recursion is forbidden, that is, the dependency graph must have no cycles.
 
 TODO: check for recursion, right now it will be infinitely loading if recursion appears.
 
-#### Merging rules
+#### Merging rules (a.k.a. what if there are values for the same key in different configs)
 
 1) If either object is null (or if either is missing / is undefined) then the other is returned
 2) If objects have different types then an error is returned
 3) If objects are dictionaries then they are merged with this algorithm
-4) otherwise the new object is returned instead of the old one (including lists)
+4) otherwise the new object is returned instead of the old one (including lists!)
 
 TODO: ^^^ describe merging better ^^^
 
@@ -317,4 +331,3 @@ configuration would look like.
         </tr>
     </tbody>
 </table>
-
diff --git a/refactor/ci.py b/refactor/ci.py
diff --git a/refactor/configs/base/common.json b/refactor/configs/base/common.json
@@ -43,14 +43,16 @@
             ],
             "extras": "caffe,kaldi,mxnet,onnx,pytorch,tensorflow,tensorflow2",
             "tests": [
-                "check_omz_tools.sh"
+                "check_omz_tools.sh@CPU",
+                "check_omz_tools.sh@GPU"
             ]
         },
         "preset_runtime": {
             "requires": [
                 "base",
                 "device_cpu",
-                "device_gpu"
+                "device_gpu",
+                "device_npu"
             ]
         },
         "preset_dev": {
@@ -59,5 +61,13 @@
                 "openvino_dev"
             ]
         }
+    },
+    "presets": {
+        "runtime": [
+            "preset_runtime"
+        ],
+        "dev": [
+            "preset_dev"
+        ]
     }
 }
diff --git a/refactor/configs/base/rhel8.json b/refactor/configs/base/rhel8.json
@@ -0,0 +1,63 @@
+{
+    "_based_on": "common",
+    "base_image": "registry.access.redhat.com/ubi8/ubi",
+    "os_id": "rhel8",
+    "components": {
+        "build_tools": {
+            "rpm": [
+                "gcc",
+                "gcc-c++",
+                "make",
+                "cmake",
+                "pkgconfig"
+            ]
+        },
+        "python": {
+            "rpm": [
+                "python39",
+                "python3-virtualenv",
+                "python3-pip"
+            ],
+            "command": "python3.9"
+        },
+        "intel-opencl-icd": {
+            "rpm": [
+                "https://repositories.intel.com/graphics/rhel/8.6/intel-opencl-22.43.24595.35-i538.el8.x86_64.rpm"
+            ],
+            "requires": [
+                "compute-runtime"
+            ]
+        },
+        "level-zero": {
+            "rpm": [
+                "https://repositories.intel.com/graphics/rhel/8.6/level-zero-1.8.8-i524.el8.x86_64.rpm"
+            ]
+        },
+        "intel-level-zero-gpu": {
+            "rpm": [
+                "https://repositories.intel.com/graphics/rhel/8.6/intel-level-zero-gpu-1.3.24595.35-i538.el8.x86_64.rpm"
+            ],
+            "requires": [
+                "compute-runtime",
+                "level-zero"
+            ]
+        },
+        "compute-runtime": {
+            "rpm": [
+                "https://repositories.intel.com/graphics/rhel/8.6/intel-gmmlib-22.3.1-i529.el8.x86_64.rpm",
+                "https://repositories.intel.com/graphics/rhel/8.6/intel-igc-core-1.0.12504.6-i537.el8.x86_64.rpm",
+                "https://repositories.intel.com/graphics/rhel/8.6/intel-igc-opencl-1.0.12504.6-i537.el8.x86_64.rpm"
+            ]
+        },
+        "opencl-loader": {
+            "rpm": [
+                "https://vault.centos.org/8-stream/AppStream/x86_64/os/Packages/ocl-icd-2.2.12-1.el8.x86_64.rpm"
+            ]
+        },
+        "device_npu": {
+            "requires": [],
+            "tests": [],
+            "_comment": "rhel8 doesn't support NPU, so we clear NPU requirements"
+        }
+    }
+}
diff --git a/refactor/configs/base/ubuntu18.json b/refactor/configs/base/ubuntu18.json
@@ -53,6 +53,11 @@
                 "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.8708/intel-igc-core_1.0.8708_amd64.deb",
                 "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.8708/intel-igc-opencl_1.0.8708_amd64.deb"
             ]
+        },
+        "device_npu": {
+            "requires": [],
+            "tests": [],
+            "_comment": "ubuntu20 doesn't support NPU, so we clear NPU requirements"
         }
     }
 }
diff --git a/refactor/configs/base/ubuntu20.json b/refactor/configs/base/ubuntu20.json
@@ -32,6 +32,11 @@
                 "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.12037.1/intel-igc-core_1.0.12037.1_amd64.deb",
                 "https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.12037.1/intel-igc-opencl_1.0.12037.1_amd64.deb"
             ]
+        },
+        "device_npu": {
+            "requires": [],
+            "tests": [],
+            "_comment": "ubuntu20 doesn't support NPU, so we clear NPU requirements"
         }
     }
 }
diff --git a/refactor/configs/releases/2023.3/ubuntu18.json b/refactor/configs/releases/2023.3/ubuntu18.json
@@ -1,16 +1,8 @@
 {
     "_based_on": "ubuntu18",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu18_2023.3.0.13775.ceeafaf64f3_x86_64.tgz",
         "version": "2023.3.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2023.3/ubuntu20.json b/refactor/configs/releases/2023.3/ubuntu20.json
@@ -1,16 +1,8 @@
 {
     "_based_on": "ubuntu20",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu20_2023.3.0.13775.ceeafaf64f3_x86_64.tgz",
         "version": "2023.3.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2023.3/ubuntu22.json b/refactor/configs/releases/2023.3/ubuntu22.json
@@ -1,16 +1,15 @@
 {
     "_based_on": "ubuntu22",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu22_2023.3.0.13775.ceeafaf64f3_x86_64.tgz",
         "version": "2023.3.0"
     },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
+    "components": {
+        "device_npu": {
+            "requires": [],
+            "tests": [],
+            "_comment": "2023* releases don't support NPU, so we clear NPU requirements"
+        }
     }
 }
diff --git a/refactor/configs/releases/2024.1/ubuntu20.json b/refactor/configs/releases/2024.1/ubuntu20.json
@@ -1,16 +1,8 @@
 {
     "_based_on": "ubuntu20",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu20_2024.1.0.15008.f4afc983258_x86_64.tgz",
         "version": "2024.1.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2024.1/ubuntu22.json b/refactor/configs/releases/2024.1/ubuntu22.json
@@ -1,18 +1,8 @@
 {
     "_based_on": "ubuntu22",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.1/linux/l_openvino_toolkit_ubuntu22_2024.1.0.15008.f4afc983258_x86_64.tgz",
         "version": "2024.1.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime",
-            "device_npu"
-        ],
-        "dev": [
-            "preset_dev",
-            "device_npu"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2024.2/ubuntu20.json b/refactor/configs/releases/2024.2/ubuntu20.json
@@ -1,16 +1,8 @@
 {
     "_based_on": "ubuntu20",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu20_2024.2.0.15519.5c0f38f83f6_x86_64.tgz",
         "version": "2024.2.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2024.2/ubuntu22.json b/refactor/configs/releases/2024.2/ubuntu22.json
@@ -1,18 +1,8 @@
 {
     "_based_on": "ubuntu22",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.2/linux/l_openvino_toolkit_ubuntu22_2024.2.0.15519.5c0f38f83f6_x86_64.tgz",
         "version": "2024.2.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime",
-            "device_npu"
-        ],
-        "dev": [
-            "preset_dev",
-            "device_npu"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2024.3/rhel8.json b/refactor/configs/releases/2024.3/rhel8.json
@@ -0,0 +1,8 @@
+{
+    "_based_on": "rhel8",
+    "_template": "Dockerfile_default.j2",
+    "package": {
+        "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_rhel8_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz",
+        "version": "2024.3.0"
+    }
+}
diff --git a/refactor/configs/releases/2024.3/ubuntu20.json b/refactor/configs/releases/2024.3/ubuntu20.json
@@ -1,16 +1,8 @@
 {
     "_based_on": "ubuntu20",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu20_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz",
         "version": "2024.3.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime"
-        ],
-        "dev": [
-            "preset_dev"
-        ]
     }
 }
diff --git a/refactor/configs/releases/2024.3/ubuntu22.json b/refactor/configs/releases/2024.3/ubuntu22.json
@@ -1,18 +1,8 @@
 {
     "_based_on": "ubuntu22",
-    "_template": "Dockerfile_2024_deb_default.j2",
+    "_template": "Dockerfile_default.j2",
     "package": {
         "url": "https://storage.openvinotoolkit.org/repositories/openvino/packages/2024.3/linux/l_openvino_toolkit_ubuntu22_2024.3.0.16041.1e3b88e4e3f_x86_64.tgz",
         "version": "2024.3.0"
-    },
-    "presets": {
-        "runtime": [
-            "preset_runtime",
-            "device_npu"
-        ],
-        "dev": [
-            "preset_dev",
-            "device_npu"
-        ]
     }
 }