Merge pull request #1 from zwh2119/main

Add baseline scheduling methods and comprehensive evaluation ways

.gitignore

@@ -736,7 +736,8 @@ model
 expr_*
 resource_data
 data_record*
-test/
+../../ResultAnalysis/pythonProject/test/
+../../ResultAnalysis/pythonProject/results/
 
 
 # End of https://www.toptal.com/developers/gitignore/api/pycharm,python,vs,macos,windows,linux,vue,vuejs

Makefile

@@ -5,6 +5,10 @@ REPOSITORY := $(or $(REPO),dayuhub)
 IMAGE_REPO ?= $(REGISTRY)/$(REPOSITORY)
 IMAGE_TAG ?= $(or $(TAG),v1.0)
 
+#$(info "REGISTRY = $(REGISTRY)")
+#$(info "REPOSITORY = $(REPOSITORY)")
+#$(info "IMAGE_TAG = $(IMAGE_TAG)")
+
 .EXPORT_ALL_VARIABLES:
 
 define BUILD_HELP_INFO
@@ -31,10 +35,18 @@ help:
 
 # Build images
 build:
-	bash ./hack/build.sh --files $(WHAT) --tag $(IMAGE_TAG) --repo $(REPOSITORY) --registry $(REGISTRY)
+	@echo "Running build images of $(WHAT)"
+	@echo "Current registry is: $(REGISTRY)"
+	@echo "Current repository is: $(REPOSITORY)"
+	@echo "Current image tag is: $(IMAGE_TAG)"
+	bash hack/make-rules/cross-build.sh --files $(WHAT) --tag $(IMAGE_TAG) --repo $(REPOSITORY) --registry $(REGISTRY)
 
 # Build all images
 all:
-	bash ./hack/build.sh --tag $(IMAGE_TAG) --repo $(REPOSITORY) --registry $(REGISTRY)
+	@echo "Running build images of $(WHAT)"
+	@echo "Current registry is: $(REGISTRY)"
+	@echo "Current repository is: $(REPOSITORY)"
+	@echo "Current image tag is: $(IMAGE_TAG)"
+	bash hack/make-rules/cross-build.sh --tag $(IMAGE_TAG) --repo $(REPOSITORY) --registry $(REGISTRY)
 
 

README.md

@@ -75,7 +75,7 @@ Meanwhile, `processor` can be equipped with user-defined application services of
 
 
 ## Quick Start
-- Install KubeEdge system on your devices ([instruction](https://box.nju.edu.cn/f/d6ba2a96a2744f0f9e0a/)). Our dayu system is based on KubeEdge.
+- Install KubeEdge system on your devices ([instruction](https://box.nju.edu.cn/f/63e12c4ea0794718b16c/)). Our dayu system is based on KubeEdge.
 
 - Modify template files in template directory '[template](template)'
 
@@ -95,6 +95,22 @@ Components in Dayu system are dependent on docker containers. Thus, if you need
 
 The official images of Dayu system is at [dockerhub/dayuhub](https://hub.docker.com/u/dayuhub).
 
+set meta information of building
+```bash
+# configure buildx buildkitd (default as empty, example at hack/resource/buildkitd_template.toml)
+vim hack/resource/buildkitd.toml
+# configure buildx driver-opt (default as empty, example at hack/resource/driver_opts_template.toml)
+vim hack/resource/driver_opts.toml
+
+# set docker meta info
+# default REG is docker.io
+# default REPO is dayuhub
+# default TAG is v1.0
+export REG=xxx
+export REPO=xxx
+export TAG=xxx
+```
+
 build all images
 ```bash
 make all

README_zh.md

@@ -71,7 +71,7 @@
 
 ## 快速开始
 
-- 在设备上安装 KubeEdge 系统（[安装指南](https://box.nju.edu.cn/f/d6ba2a96a2744f0f9e0a/)）
+- 在设备上安装 KubeEdge 系统（[安装指南](https://box.nju.edu.cn/f/63e12c4ea0794718b16c/)）
 - 修改模板目录中的模板文件（'[template](template)'）
 - 根据模板中的设置将文件部署到设备上，示例部署文件在[这里](https://box.nju.edu.cn/d/0dcaabb5362c4dfc8008/)
 - 安装/卸载大禹系统
@@ -89,6 +89,22 @@ ACTION=stop TEMPLATE=template/ bash - dayu.sh
 
 大禹系统的官方镜像托管在[dockerhub/dayuhub](https://hub.docker.com/u/dayuhub)
 
+设置构建镜像元信息
+```bash
+# 配置 buildx buildkitd (默认为空, 样例可参照 hack/resource/buildkitd_template.toml)
+vim hack/resource/buildkitd.toml
+# 配置 buildx driver-opt (默认为空, 样例可参照 hack/resource/driver_opts_template.toml)
+vim hack/resource/driver_opts.toml
+
+# 设置docker参数
+# default REG is docker.io
+# default REPO is dayuhub
+# default TAG is v1.0
+export REG=xxx
+export REPO=xxx
+export TAG=xxx
+```
+
 构建所有镜像
 ```bash
 make all

backend/backend_core.py

@@ -1,14 +1,14 @@
-import json
+import cv2
 import numpy as np
 import os
 import time
 from core.lib.content import Task
-from core.lib.common import LOGGER, Context, YamlOps, FileOps, Counter, SystemConstant
+from core.lib.common import LOGGER, Context, YamlOps, FileOps, Counter, SystemConstant, EncodeOps
 from core.lib.network import http_request, NodeInfo, PortInfo, get_merge_address, NetworkAPIPath, NetworkAPIMethod
 
 from kube_helper import KubeHelper
 from template_helper import TemplateHelper
-from utils import get_first_frame_from_video, encode_image, draw_bboxes
+from utils import get_first_frame_from_video, draw_bboxes
 
 
 class BackendCore:
@@ -44,6 +44,8 @@ def __init__(self):
         self.save_yaml_path = 'resources.yaml'
         self.log_file_path = 'log.json'
 
+        self.default_visualization_image = 'default_visualization.png'
+
         self.parse_base_info()
 
     def parse_base_info(self):
@@ -56,12 +58,19 @@ def parse_base_info(self):
         except KeyError as e:
             LOGGER.warning(f'Parse base info failed: {str(e)}')
 
+    def get_log_file_name(self):
+        base_info = self.template_helper.load_base_info()
+        load_file_name = base_info['log-file-name']
+        if not load_file_name:
+            return None
+        return load_file_name.split('.')[0]
+
     def parse_apply_templates(self, policy, source_deploy):
         yaml_dict = {}
 
         yaml_dict.update(self.template_helper.load_policy_apply_yaml(policy))
 
-        service_dict = self.extract_service_from_source_deployment(source_deploy)
+        service_dict, source_deploy = self.extract_service_from_source_deployment(source_deploy)
         yaml_dict.update({'processor': self.template_helper.load_application_apply_yaml(service_dict)})
 
         docs_list = self.template_helper.finetune_yaml_parameters(yaml_dict, source_deploy)
@@ -77,14 +86,18 @@ def extract_service_from_source_deployment(self, source_deploy):
         for s in source_deploy:
             pipeline = s['pipeline']
             node = s['node']
+            extracted_pipeline = []
             for service_id in pipeline:
                 service = self.find_service_by_id(service_id)
+                service_name = service['service']
+                service_yaml = service['yaml']
+                extracted_pipeline.append(service)
                 if service_id in service_dict:
                     service_dict[service_id]['node'].append(node)
                 else:
-                    service_dict[service_id] = {'yaml': service['yaml'], 'node': [node]}
-
-        return service_dict
+                    service_dict[service_id] = {'service_name': service_name, 'yaml': service_yaml, 'node': [node]}
+            s['pipeline'] = extracted_pipeline
+        return service_dict, source_deploy
 
     def get_yaml_docs(self):
         if self.cur_yaml_docs:
@@ -224,9 +237,11 @@ def run_get_result(self):
                     image = get_first_frame_from_video(file_path)
                     image = draw_bboxes(image, content[0][0])
 
-                    base64_data = encode_image(image)
+                    base64_data = EncodeOps.encode_image(image)
                 except Exception as e:
-                    base64_data = bytes('', encoding='utf8')
+                    base64_data = EncodeOps.encode_image(
+                        cv2.imread(self.default_visualization_image)
+                    )
                     LOGGER.warning(f'Video visualization fetch failed: {str(e)}')
                 if os.path.exists(file_path):
                     os.remove(file_path)
@@ -243,7 +258,7 @@ def run_get_result(self):
 
     def check_datasource_config(self, config_path):
         if not YamlOps.is_yaml_file(config_path):
-            return False
+            return None
 
         config = YamlOps.read_yaml(config_path)
         try:

backend/backend_server.py

@@ -448,6 +448,7 @@ def install_loop(yaml_docs):
             yaml = self.server.parse_apply_templates(policy, source_deploy)
         except Exception as e:
             LOGGER.warning(f'Parse templates failed: {str(e)}')
+            LOGGER.exception(e)
             yaml = None
         try:
             result = install_loop(yaml)
@@ -621,13 +622,17 @@ async def download_log(self, backtask: BackgroundTasks):
         :return:
         file
         """
-        formatted_time = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+        file_name = self.server.get_log_file_name()
+        if not file_name:
+            formatted_time = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+            file_name = f'LOG_SYSTEM_DAYU_NAMESPACE_{self.server.namespace}_TIME_{formatted_time}'
+
         log_content = self.server.download_log_file()
         with open(self.server.log_file_path, 'w') as f:
             json.dump(log_content, f)
         backtask.add_task(FileOps.remove_file, self.server.log_file_path)
         return FileResponse(
             path=self.server.log_file_path,
-            filename=f'LOG_SYSTEM_DAYU_NAMESPACE_{self.server.namespace}_TIME_{formatted_time}.json',
+            filename=f'{file_name}.json',
             background=backtask.add_task(FileOps.remove_file, self.server.log_file_path)
         )

backend/template_helper.py

@@ -170,7 +170,7 @@ def finetune_generator_yaml(self, yaml_doc, source_deploy):
                     {'name': 'SOURCE_TYPE', 'value': str(source['source_type'])},
                     {'name': 'SOURCE_ID', 'value': str(source['id'])},
                     {'name': 'SOURCE_METADATA', 'value': str(source['metadata'])},
-                    {'name': 'PIPELINE', 'value': str([{'service_name': service} for service in pipeline])},
+                    {'name': 'PIPELINE', 'value': str([{'service_name': service['service']} for service in pipeline])},
 
                 ])
 
@@ -261,7 +261,8 @@ def finetune_processor_yaml(self, service_dict, cloud_node):
         yaml_docs = []
         for index, service_id in enumerate(service_dict):
             yaml_doc = service_dict[service_id]['service']
-            yaml_doc = self.fill_template(yaml_doc, f'processor-{service_id}')
+            service_name = service_dict[service_id]['service_name']
+            yaml_doc = self.fill_template(yaml_doc, f'processor-{service_name}')
 
             edge_nodes = service_dict[service_id]['node']
 

backend/utils.py

@@ -1,5 +1,4 @@
 import cv2
-import base64
 import numpy as np
 
 
@@ -37,65 +36,6 @@ def get_first_frame_from_video(video_path):
     return frame
 
 
-def encode_image(image):
-    """
-    Encodes an image to a base64 string with a data URL prefix.
-
-    Parameters:
-        image (numpy.ndarray): The image to encode, typically read by OpenCV (in BGR format).
-
-    Returns:
-        str: A base64-encoded string representing the image, prefixed with "data:image/jpg;base64,".
-
-    Raises:
-        ValueError: If the image encoding fails or the input is not a valid numpy array.
-    """
-    if not isinstance(image, np.ndarray):
-        raise ValueError("Input image must be a numpy array.")
-
-    # Encode the image to JPEG format
-    success, buffer = cv2.imencode('.jpg', image)
-    if not success:
-        raise ValueError("Image encoding failed.")
-
-    # Convert the encoded image to base64
-    base64_str = base64.b64encode(buffer).decode('utf-8')
-    return f"data:image/jpg;base64,{base64_str}"
-
-
-def decode_image(encoded_img_str):
-    """
-    Decodes a base64-encoded image string back into an OpenCV image (numpy array).
-
-    Parameters:
-        encoded_img_str (str): The base64 string representing the image.
-                               It should include the "data:image/jpg;base64," prefix.
-
-    Returns:
-        numpy.ndarray: The decoded image in BGR format suitable for OpenCV processing.
-
-    Raises:
-        ValueError: If decoding fails due to an invalid base64 string or incompatible data.
-    """
-    if not isinstance(encoded_img_str, str):
-        raise ValueError("Encoded image must be a string.")
-
-    # Remove the data URL prefix if present
-    if encoded_img_str.startswith("data:image"):
-        encoded_img_str = encoded_img_str.split(',', 1)[1]
-
-    try:
-        # Decode the base64 string to bytes
-        encoded_img_bytes = base64.b64decode(encoded_img_str)
-        # Convert bytes to a numpy array and decode to an OpenCV image
-        decoded_img = cv2.imdecode(np.frombuffer(encoded_img_bytes, np.uint8), cv2.IMREAD_COLOR)
-        if decoded_img is None:
-            raise ValueError("Image decoding failed; data may be corrupt or incompatible.")
-        return decoded_img
-    except (base64.binascii.Error, ValueError) as e:
-        raise ValueError("Invalid base64 string or incompatible image data.") from e
-
-
 def draw_bboxes(frame, bboxes):
     """
     Draws bounding boxes on an image frame.

build/generator.Dockerfile

@@ -17,6 +17,7 @@ ENV TZ=Asia/Shanghai
 # Install python3
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
+      ffmpeg \
       python3 \
       python3-pip \
       python3-dev \

config/datasource_configs/road.yaml

@@ -6,7 +6,7 @@ source_list:
     url: ""
     dir: "road_0/"
     metadata:
-      resolution: "1080p"
+      resolution: "540p"
       fps: 30
       encoding: "mp4v"
       buffer_size: 4

config/datasource_configs/road_and_street.yaml

@@ -6,7 +6,7 @@ source_list:
     url: ""
     dir: "road_0/"
     metadata:
-      resolution: "1080p"
+      resolution: "540p"
       fps: 30
       encoding: "mp4v"
       buffer_size: 4

config/datasource_configs/road_dense.yaml

@@ -0,0 +1,13 @@
+source_name: "Road Dense HTTP"
+source_type: "video"
+source_mode: "http_video"
+source_list:
+  - name: "road_camera_dense"
+    url: ""
+    dir: "road_dense/"
+    metadata:
+      resolution: "540p"
+      fps: 30
+      encoding: "mp4v"
+      buffer_size: 4
+

config/datasource_configs/road_sparse.yaml

@@ -0,0 +1,13 @@
+source_name: "Road Sparse HTTP"
+source_type: "video"
+source_mode: "http_video"
+source_list:
+  - name: "road_camera_sparse"
+    url: ""
+    dir: "road_sparse/"
+    metadata:
+      resolution: "540p"
+      fps: 30
+      encoding: "mp4v"
+      buffer_size: 4
+

config/datasource_configs/road_two.yaml

@@ -6,7 +6,7 @@ source_list:
     url: ""
     dir: "road_0/"
     metadata:
-      resolution: "1080p"
+      resolution: "540p"
       fps: 30
       encoding: "mp4v"
       buffer_size: 4

config/datasource_configs/street_dense.yaml

@@ -0,0 +1,14 @@
+source_name: "Street Dense HTTP"
+source_type: "video"
+source_mode: "http_video"
+source_list:
+  - name: "street_camera_dense"
+    url: ""
+    dir: "street_dense"
+    metadata:
+      resolution: "1080p"
+      fps: 30
+      encoding: "mp4v"
+      buffer_size: 4
+
+